From 4110b730dbee59f5515a0bf9997b6cd167965080 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 1 Oct 2013 20:30:04 +0200 Subject: [PATCH 001/273] x86[_64]: Add support for CPUID sub-leaves Some CPUID leaves may contain one or more sub-leaves accessed by setting ECX to an appropriate value. --- .../private/kernel/arch/x86/arch_system_info.h | 2 +- src/add-ons/kernel/cpu/x86/generic_x86.cpp | 4 ++-- .../power/x86_cpuidle/intel_cpuidle.cpp | 2 +- src/system/boot/platform/bios_ia32/cpu.cpp | 2 +- src/system/boot/platform/bios_ia32/long.cpp | 2 +- src/system/boot/platform/bios_ia32/smp.cpp | 4 ++-- src/system/kernel/arch/x86/32/cpuid.S | 4 +++- src/system/kernel/arch/x86/64/cpuid.S | 4 +++- src/system/kernel/arch/x86/arch_cpu.cpp | 18 +++++++++--------- .../kernel/arch/x86/arch_system_info.cpp | 2 +- 10 files changed, 24 insertions(+), 20 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_system_info.h b/headers/private/kernel/arch/x86/arch_system_info.h index b8a7383de9..0f92e3799f 100644 --- a/headers/private/kernel/arch/x86/arch_system_info.h +++ b/headers/private/kernel/arch/x86/arch_system_info.h @@ -12,7 +12,7 @@ extern "C" { #endif -status_t get_current_cpuid(cpuid_info* info, uint32 eax); +status_t get_current_cpuid(cpuid_info* info, uint32 eax, uint32 ecx); uint32 get_eflags(void); void set_eflags(uint32 value); diff --git a/src/add-ons/kernel/cpu/x86/generic_x86.cpp b/src/add-ons/kernel/cpu/x86/generic_x86.cpp index b394ff9b66..94e6a1b853 100644 --- a/src/add-ons/kernel/cpu/x86/generic_x86.cpp +++ b/src/add-ons/kernel/cpu/x86/generic_x86.cpp @@ -208,9 +208,9 @@ generic_mtrr_compute_physical_mask(void) uint32 bits = 36; cpuid_info cpuInfo; - if (get_current_cpuid(&cpuInfo, 0x80000000) == B_OK + if (get_current_cpuid(&cpuInfo, 0x80000000, 0) == B_OK && (cpuInfo.eax_0.max_eax & 0xff) >= 8) { - get_current_cpuid(&cpuInfo, 0x80000008); + get_current_cpuid(&cpuInfo, 0x80000008, 0); bits = cpuInfo.regs.eax & 0xff; // Obviously, the bits are not always reported correctly diff --git a/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp b/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp index 7f6fbd6c4b..cbbd1bc988 100644 --- a/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp +++ b/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp @@ -91,7 +91,7 @@ intel_cpuidle_init(void) return B_ERROR; cpuid_info cpuid; - get_current_cpuid(&cpuid, 5); + get_current_cpuid(&cpuid, 5, 0); /* ecx[0] monitor/mwait extension supported * ecx[1] support for treating interrupts as break-events for mwait * edx number of sub-states diff --git a/src/system/boot/platform/bios_ia32/cpu.cpp b/src/system/boot/platform/bios_ia32/cpu.cpp index e9d405b883..cc70cbaba9 100644 --- a/src/system/boot/platform/bios_ia32/cpu.cpp +++ b/src/system/boot/platform/bios_ia32/cpu.cpp @@ -327,7 +327,7 @@ check_cpu_features() } cpuid_info info; - if (get_current_cpuid(&info, 1) != B_OK) + if (get_current_cpuid(&info, 1, 0) != B_OK) return B_ERROR; if ((info.eax_1.features & RDTSC_FEATURE) == 0) { diff --git a/src/system/boot/platform/bios_ia32/long.cpp b/src/system/boot/platform/bios_ia32/long.cpp index d632efc970..3db5220e36 100644 --- a/src/system/boot/platform/bios_ia32/long.cpp +++ b/src/system/boot/platform/bios_ia32/long.cpp @@ -330,7 +330,7 @@ long_start_kernel() { // Check whether long mode is supported. cpuid_info info; - get_current_cpuid(&info, 0x80000001); + get_current_cpuid(&info, 0x80000001, 0); if ((info.regs.edx & (1 << 29)) == 0) panic("64-bit kernel requires a 64-bit CPU"); diff --git a/src/system/boot/platform/bios_ia32/smp.cpp b/src/system/boot/platform/bios_ia32/smp.cpp index 138e1cd21a..a0ed686415 100644 --- a/src/system/boot/platform/bios_ia32/smp.cpp +++ b/src/system/boot/platform/bios_ia32/smp.cpp @@ -581,7 +581,7 @@ smp_add_safemode_menus(Menu *menu) item->SetHelpText("Disables using the local APIC, also disables SMP."); cpuid_info info; - if (get_current_cpuid(&info, 1) == B_OK + if (get_current_cpuid(&info, 1, 0) == B_OK && (info.regs.ecx & IA32_FEATURE_EXT_X2APIC) != 0) { #if 0 menu->AddItem(item = new(nothrow) MenuItem("Disable X2APIC")); @@ -617,7 +617,7 @@ smp_init(void) #endif cpuid_info info; - if (get_current_cpuid(&info, 1) != B_OK) + if (get_current_cpuid(&info, 1, 0) != B_OK) return; if ((info.eax_1.features & IA32_FEATURE_APIC) == 0) { diff --git a/src/system/kernel/arch/x86/32/cpuid.S b/src/system/kernel/arch/x86/32/cpuid.S index 5eb25731db..0595a4d4f7 100644 --- a/src/system/kernel/arch/x86/32/cpuid.S +++ b/src/system/kernel/arch/x86/32/cpuid.S @@ -11,12 +11,14 @@ .text -/* void get_current_cpuid(cpuid_info *info, uint32 eaxRegister) */ +/* void get_current_cpuid(cpuid_info *info, uint32 eaxRegister, + uint32 ecxRegister) */ FUNCTION(get_current_cpuid): pushl %ebx pushl %edi movl 12(%esp),%edi /* first arg points to the cpuid_info structure */ movl 16(%esp),%eax /* second arg sets up eax */ + movl 20(%esp),%ecx /* third arg sets up ecx */ cpuid movl %eax,0(%edi) /* copy the regs into the cpuid_info structure */ movl %ebx,4(%edi) diff --git a/src/system/kernel/arch/x86/64/cpuid.S b/src/system/kernel/arch/x86/64/cpuid.S index 8e6b07cd6b..1085359a1f 100644 --- a/src/system/kernel/arch/x86/64/cpuid.S +++ b/src/system/kernel/arch/x86/64/cpuid.S @@ -10,10 +10,12 @@ .text -/* status_t get_current_cpuid(cpuid_info* info, uint32 eaxRegister) */ +/* status_t get_current_cpuid(cpuid_info* info, uint32 eaxRegister, + uint32 ecxRegister) */ FUNCTION(get_current_cpuid): push %rbx movl %esi, %eax + movl %edx, %ecx cpuid movl %eax, 0(%rdi) movl %ebx, 4(%rdi) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index a817ec05b7..6ad0bc7e66 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -522,7 +522,7 @@ detect_cpu(int currentCPU) cpu->arch.model_name[0] = 0; // print some fun data - get_current_cpuid(&cpuid, 0); + get_current_cpuid(&cpuid, 0, 0); uint32 maxBasicLeaf = cpuid.eax_0.max_eax; // build the vendor string @@ -530,7 +530,7 @@ detect_cpu(int currentCPU) memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id)); // get the family, model, stepping - get_current_cpuid(&cpuid, 1); + get_current_cpuid(&cpuid, 1, 0); cpu->arch.type = cpuid.eax_1.type; cpu->arch.family = cpuid.eax_1.family; cpu->arch.extended_family = cpuid.eax_1.extended_family; @@ -561,27 +561,27 @@ detect_cpu(int currentCPU) } // see if we can get the model name - get_current_cpuid(&cpuid, 0x80000000); + get_current_cpuid(&cpuid, 0x80000000, 0); uint32 maxExtendedLeaf = cpuid.eax_0.max_eax; if (maxExtendedLeaf >= 0x80000004) { // build the model string (need to swap ecx/edx data before copying) unsigned int temp; memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name)); - get_current_cpuid(&cpuid, 0x80000002); + get_current_cpuid(&cpuid, 0x80000002, 0); temp = cpuid.regs.edx; cpuid.regs.edx = cpuid.regs.ecx; cpuid.regs.ecx = temp; memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars)); - get_current_cpuid(&cpuid, 0x80000003); + get_current_cpuid(&cpuid, 0x80000003, 0); temp = cpuid.regs.edx; cpuid.regs.edx = cpuid.regs.ecx; cpuid.regs.ecx = temp; memcpy(cpu->arch.model_name + 16, cpuid.as_chars, sizeof(cpuid.as_chars)); - get_current_cpuid(&cpuid, 0x80000004); + get_current_cpuid(&cpuid, 0x80000004, 0); temp = cpuid.regs.edx; cpuid.regs.edx = cpuid.regs.ecx; cpuid.regs.ecx = temp; @@ -604,19 +604,19 @@ detect_cpu(int currentCPU) } // load feature bits - get_current_cpuid(&cpuid, 1); + get_current_cpuid(&cpuid, 1, 0); cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx if (maxExtendedLeaf >= 0x80000001) { - get_current_cpuid(&cpuid, 0x80000001); + get_current_cpuid(&cpuid, 0x80000001, 0); cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx if (cpu->arch.vendor != VENDOR_AMD) cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; } if (maxBasicLeaf >= 6) { - get_current_cpuid(&cpuid, 6); + get_current_cpuid(&cpuid, 6, 0); cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; } diff --git a/src/system/kernel/arch/x86/arch_system_info.cpp b/src/system/kernel/arch/x86/arch_system_info.cpp index a1423ffaa7..280ca0eb65 100644 --- a/src/system/kernel/arch/x86/arch_system_info.cpp +++ b/src/system/kernel/arch/x86/arch_system_info.cpp @@ -29,7 +29,7 @@ get_cpuid_for(cpuid_info *info, uint32 currentCPU, uint32 eaxRegister, if (currentCPU != forCPU) return false; - get_current_cpuid(info, eaxRegister); + get_current_cpuid(info, eaxRegister, 0); return true; } From 8ec897323ec1c6b6d2ed7e540d6118fbd3dd9855 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 01:19:17 +0200 Subject: [PATCH 002/273] x86[_64]: Add CPU topology detection for Intel processors --- headers/private/kernel/cpu.h | 12 ++ src/system/kernel/arch/x86/arch_cpu.cpp | 195 ++++++++++++++++++++++++ 2 files changed, 207 insertions(+) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 850e666913..ab2a850d03 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -31,6 +31,15 @@ namespace BKernel { using BKernel::Thread; +typedef enum cpu_topology_level { + CPU_TOPOLOGY_SMT, + CPU_TOPOLOGY_CORE, + CPU_TOPOLOGY_PACKAGE, + // + CPU_TOPOLOGY_LEVELS +} cpu_topology_level; + + /* CPU local data structure */ typedef struct cpu_ent { @@ -56,6 +65,9 @@ typedef struct cpu_ent { bool invoke_scheduler_if_idle; bool disabled; + // CPU topology information + int topology_id[CPU_TOPOLOGY_LEVELS]; + // arch-specific stuff arch_cpu_info arch; } cpu_ent __attribute__((aligned(64))); diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 6ad0bc7e66..23b6979634 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -34,6 +34,7 @@ #define DUMP_FEATURE_STRING 1 +#define DUMP_CPU_TOPOLOGY 1 /* cpu vendor info */ @@ -122,6 +123,37 @@ x86_optimized_functions gOptimizedFunctions = { &memset_generic_end }; +/* CPU topology information */ +static uint32 (*getCPUTopologyID)(int currentCPU) = NULL; +static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; +static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; + + +// http://graphics.stanford.edu/~seander/bithacks.html +static inline uint32 +nextPowerOf2(uint32 v) +{ + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + + return v; +} + + +// http://graphics.stanford.edu/~seander/bithacks.html +static inline uint32 +countSetBits(uint32 v) +{ + v = v - ((v >> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + return (((v + (v >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24; +} + static status_t acpi_shutdown(bool rebootSystem) @@ -506,6 +538,167 @@ dump_feature_string(int currentCPU, cpu_ent* cpu) #endif // DUMP_FEATURE_STRING +static uint32 +getIntelCPUInitialx2APICID(int currentCPU) +{ + (void)currentCPU; + + cpuid_info cpuid; + get_current_cpuid(&cpuid, 11, 0); + return cpuid.regs.edx; +} + + +static inline status_t +detectIntelCPUTopologyx2APIC(int maxBasicLeaf) +{ + if (maxBasicLeaf < 11) + return B_UNSUPPORTED; + + const int kLevelCount = CPU_TOPOLOGY_LEVELS; + uint8 hierarchyLevels[kLevelCount]; + + int currentLevel = 0; + int levelType; + int levelsSet = 0; + + do { + cpuid_info cpuid; + get_current_cpuid(&cpuid, 11, currentLevel); + if (currentLevel == 0 && cpuid.regs.ebx == 0) + return B_UNSUPPORTED; + + levelType = (cpuid.regs.ecx >> 8) & 0xff; + switch (levelType) { + case 1: // SMT + hierarchyLevels[CPU_TOPOLOGY_SMT] = cpuid.regs.eax & 0x1f; + levelsSet |= 1; + break; + case 2: // core + hierarchyLevels[CPU_TOPOLOGY_CORE] = cpuid.regs.eax & 0x1f; + levelsSet |= 2; + break; + } + + currentLevel++; + } while(levelType != 0 && levelsSet != 3); + + getCPUTopologyID = getIntelCPUInitialx2APICID; + + for (int i = 0; i < kLevelCount; i++) { + uint32 mask = ~uint32(0); + if (i < kLevelCount - 1) + mask = (1 << hierarchyLevels[i]) - 1; + if (i > 0) + mask &= ~sHierarchyMask[i - 1]; + sHierarchyMask[i] = mask; + sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0; + } + + return B_OK; +} + + +static uint32 +getIntelCPULegacyInitialAPICID(int currentCPU) +{ + (void)currentCPU; + + cpuid_info cpuid; + get_current_cpuid(&cpuid, 1, 0); + return cpuid.regs.ebx >> 24; +} + + +static inline void +detectIntelCPUTopologyLegacy(int maxBasicLeaf) +{ + getCPUTopologyID = getIntelCPULegacyInitialAPICID; + + cpuid_info cpuid; + + get_current_cpuid(&cpuid, 1, 0); + int maxLogicalID = nextPowerOf2((cpuid.regs.ebx >> 16) & 0xff); + + int maxCoreID = 1; + if (maxBasicLeaf >= 4) { + get_current_cpuid(&cpuid, 4, 0); + maxCoreID = nextPowerOf2((cpuid.regs.eax >> 26) + 1); + } + + ASSERT(maxLogicalID >= maxCoreID); + const int kMaxSMTID = maxLogicalID / maxCoreID; + + sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; + sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; + + sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; + sHierarchyShift[CPU_TOPOLOGY_CORE] + = countSetBits(sHierarchyShift[CPU_TOPOLOGY_SMT]); + + const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] + | sHierarchyMask[CPU_TOPOLOGY_CORE]; + sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; + sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = countSetBits(kSinglePackageMask); +} + + +static uint32 +getSimpleCPUTopologyID(int currentCPU) +{ + return currentCPU; +} + + +static inline int +getTopologyLevelID(uint32 id, cpu_topology_level level) +{ + ASSERT(level < CPU_TOPOLOGY_LEVELS); + return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; +} + + +static void +detectCPUTopology(int currentCPU, cpu_ent* cpu, int maxBasicLeaf) +{ + if (currentCPU == 0) { + if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { + if (cpu->arch.vendor == VENDOR_INTEL) { + status_t result = detectIntelCPUTopologyx2APIC(maxBasicLeaf); + if (result != B_OK) + detectIntelCPUTopologyLegacy(maxBasicLeaf); + return; + } + } + + dprintf("No CPU topology information available.\n"); + + getCPUTopologyID = getSimpleCPUTopologyID; + + memset(sHierarchyMask, 0, sizeof(sHierarchyMask)); + sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); + + memset(sHierarchyShift, 0, sizeof(sHierarchyShift)); + } + + ASSERT(getCPUTopologyID != NULL); + int topologyID = getCPUTopologyID(currentCPU); + cpu->topology_id[CPU_TOPOLOGY_SMT] + = getTopologyLevelID(topologyID, CPU_TOPOLOGY_SMT); + cpu->topology_id[CPU_TOPOLOGY_CORE] + = getTopologyLevelID(topologyID, CPU_TOPOLOGY_CORE); + cpu->topology_id[CPU_TOPOLOGY_PACKAGE] + = getTopologyLevelID(topologyID, CPU_TOPOLOGY_PACKAGE); + +#if DUMP_CPU_TOPOLOGY + dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, + topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], + cpu->topology_id[CPU_TOPOLOGY_CORE], + cpu->topology_id[CPU_TOPOLOGY_SMT]); +#endif +} + + static void detect_cpu(int currentCPU) { @@ -621,6 +814,8 @@ detect_cpu(int currentCPU) cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; } + detectCPUTopology(currentCPU, cpu, maxBasicLeaf); + #if DUMP_FEATURE_STRING dump_feature_string(currentCPU, cpu); #endif From fafeda52eacdec6babbb789ad214a91d6387d38e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 01:49:10 +0200 Subject: [PATCH 003/273] x86[_64]: Do not return too soon from detectCPUTopology() --- src/system/kernel/arch/x86/arch_cpu.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 23b6979634..7b7992d31d 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -610,7 +610,7 @@ getIntelCPULegacyInitialAPICID(int currentCPU) } -static inline void +static inline status_t detectIntelCPUTopologyLegacy(int maxBasicLeaf) { getCPUTopologyID = getIntelCPULegacyInitialAPICID; @@ -640,6 +640,8 @@ detectIntelCPUTopologyLegacy(int maxBasicLeaf) | sHierarchyMask[CPU_TOPOLOGY_CORE]; sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = countSetBits(kSinglePackageMask); + + return B_OK; } @@ -662,23 +664,25 @@ static void detectCPUTopology(int currentCPU, cpu_ent* cpu, int maxBasicLeaf) { if (currentCPU == 0) { + status_t result = B_UNSUPPORTED; if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { if (cpu->arch.vendor == VENDOR_INTEL) { - status_t result = detectIntelCPUTopologyx2APIC(maxBasicLeaf); + result = detectIntelCPUTopologyx2APIC(maxBasicLeaf); if (result != B_OK) - detectIntelCPUTopologyLegacy(maxBasicLeaf); - return; + result = detectIntelCPUTopologyLegacy(maxBasicLeaf); } } - dprintf("No CPU topology information available.\n"); + if (result != B_OK) { + dprintf("No CPU topology information available.\n"); - getCPUTopologyID = getSimpleCPUTopologyID; + getCPUTopologyID = getSimpleCPUTopologyID; - memset(sHierarchyMask, 0, sizeof(sHierarchyMask)); - sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); + memset(sHierarchyMask, 0, sizeof(sHierarchyMask)); + sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); - memset(sHierarchyShift, 0, sizeof(sHierarchyShift)); + memset(sHierarchyShift, 0, sizeof(sHierarchyShift)); + } } ASSERT(getCPUTopologyID != NULL); From f1644d9d0b31e411a43c73afc4fd46bf73718f8c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 01:54:15 +0200 Subject: [PATCH 004/273] x86[_64]: Set level shift by counting bits in mask --- src/system/kernel/arch/x86/arch_cpu.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 7b7992d31d..614d45e0e5 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -634,7 +634,7 @@ detectIntelCPUTopologyLegacy(int maxBasicLeaf) sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; sHierarchyShift[CPU_TOPOLOGY_CORE] - = countSetBits(sHierarchyShift[CPU_TOPOLOGY_SMT]); + = countSetBits(sHierarchyMask[CPU_TOPOLOGY_SMT]); const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] | sHierarchyMask[CPU_TOPOLOGY_CORE]; From c9b6f27d949a38ad0a02d55d29cc4dbe7b229069 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 02:34:35 +0200 Subject: [PATCH 005/273] x86[_64]: Add CPU topology detection for AMD processors --- src/system/kernel/arch/x86/arch_cpu.cpp | 97 ++++++++++++++++++------- 1 file changed, 69 insertions(+), 28 deletions(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 614d45e0e5..e7cf2cb592 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -538,6 +538,68 @@ dump_feature_string(int currentCPU, cpu_ent* cpu) #endif // DUMP_FEATURE_STRING +static void +computeCPUHierarchyMasks(int maxLogicalID, int maxCoreID) +{ + ASSERT(maxLogicalID >= maxCoreID); + const int kMaxSMTID = maxLogicalID / maxCoreID; + + sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; + sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; + + sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; + sHierarchyShift[CPU_TOPOLOGY_CORE] + = countSetBits(sHierarchyMask[CPU_TOPOLOGY_SMT]); + + const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] + | sHierarchyMask[CPU_TOPOLOGY_CORE]; + sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; + sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = countSetBits(kSinglePackageMask); +} + + +static uint32 +getCPULegacyInitialAPICID(int currentCPU) +{ + (void)currentCPU; + + cpuid_info cpuid; + get_current_cpuid(&cpuid, 1, 0); + return cpuid.regs.ebx >> 24; +} + + +static inline status_t +detectAMDCPUTopology(int maxBasicLeaf, int maxExtendedLeaf) +{ + getCPUTopologyID = getCPULegacyInitialAPICID; + + cpuid_info cpuid; + get_current_cpuid(&cpuid, 1, 0); + int maxLogicalID = nextPowerOf2((cpuid.regs.ebx >> 16) & 0xff); + + int maxCoreID = 1; + if (maxExtendedLeaf >= 0x80000008) { + get_current_cpuid(&cpuid, 0x80000008, 0); + maxCoreID = (cpuid.regs.ecx >> 12) & 0xf; + if (maxCoreID != 0) + maxCoreID = 1 << maxCoreID; + else + maxCoreID = nextPowerOf2((cpuid.regs.edx & 0xf) + 1); + } + + if (maxExtendedLeaf >= 0x80000001) { + get_current_cpuid(&cpuid, 0x80000001, 0); + if ((cpuid.regs.ecx & 2) != 0) + maxCoreID = maxLogicalID; + } + + computeCPUHierarchyMasks(maxLogicalID, maxCoreID); + + return B_OK; +} + + static uint32 getIntelCPUInitialx2APICID(int currentCPU) { @@ -599,21 +661,10 @@ detectIntelCPUTopologyx2APIC(int maxBasicLeaf) } -static uint32 -getIntelCPULegacyInitialAPICID(int currentCPU) -{ - (void)currentCPU; - - cpuid_info cpuid; - get_current_cpuid(&cpuid, 1, 0); - return cpuid.regs.ebx >> 24; -} - - static inline status_t detectIntelCPUTopologyLegacy(int maxBasicLeaf) { - getCPUTopologyID = getIntelCPULegacyInitialAPICID; + getCPUTopologyID = getCPULegacyInitialAPICID; cpuid_info cpuid; @@ -626,20 +677,7 @@ detectIntelCPUTopologyLegacy(int maxBasicLeaf) maxCoreID = nextPowerOf2((cpuid.regs.eax >> 26) + 1); } - ASSERT(maxLogicalID >= maxCoreID); - const int kMaxSMTID = maxLogicalID / maxCoreID; - - sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1; - sHierarchyShift[CPU_TOPOLOGY_SMT] = 0; - - sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; - sHierarchyShift[CPU_TOPOLOGY_CORE] - = countSetBits(sHierarchyMask[CPU_TOPOLOGY_SMT]); - - const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] - | sHierarchyMask[CPU_TOPOLOGY_CORE]; - sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; - sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = countSetBits(kSinglePackageMask); + computeCPUHierarchyMasks(maxLogicalID, maxCoreID); return B_OK; } @@ -661,11 +699,14 @@ getTopologyLevelID(uint32 id, cpu_topology_level level) static void -detectCPUTopology(int currentCPU, cpu_ent* cpu, int maxBasicLeaf) +detectCPUTopology(int currentCPU, cpu_ent* cpu, int maxBasicLeaf, + int maxExtendedLeaf) { if (currentCPU == 0) { status_t result = B_UNSUPPORTED; if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { + if (cpu->arch.vendor == VENDOR_AMD) + result = detectAMDCPUTopology(maxBasicLeaf, maxExtendedLeaf); if (cpu->arch.vendor == VENDOR_INTEL) { result = detectIntelCPUTopologyx2APIC(maxBasicLeaf); if (result != B_OK) @@ -818,7 +859,7 @@ detect_cpu(int currentCPU) cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; } - detectCPUTopology(currentCPU, cpu, maxBasicLeaf); + detectCPUTopology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); #if DUMP_FEATURE_STRING dump_feature_string(currentCPU, cpu); From fa6f78aee77ab0cdb60af20dbf479218afeb8d33 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 21:03:34 +0200 Subject: [PATCH 006/273] x86[_64]: Use uint32 for maximum CPUID leaf number --- src/system/kernel/arch/x86/arch_cpu.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index e7cf2cb592..84b37f2e05 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -570,7 +570,7 @@ getCPULegacyInitialAPICID(int currentCPU) static inline status_t -detectAMDCPUTopology(int maxBasicLeaf, int maxExtendedLeaf) +detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) { getCPUTopologyID = getCPULegacyInitialAPICID; @@ -612,7 +612,7 @@ getIntelCPUInitialx2APICID(int currentCPU) static inline status_t -detectIntelCPUTopologyx2APIC(int maxBasicLeaf) +detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) { if (maxBasicLeaf < 11) return B_UNSUPPORTED; @@ -662,7 +662,7 @@ detectIntelCPUTopologyx2APIC(int maxBasicLeaf) static inline status_t -detectIntelCPUTopologyLegacy(int maxBasicLeaf) +detectIntelCPUTopologyLegacy(uint32 maxBasicLeaf) { getCPUTopologyID = getCPULegacyInitialAPICID; @@ -699,8 +699,8 @@ getTopologyLevelID(uint32 id, cpu_topology_level level) static void -detectCPUTopology(int currentCPU, cpu_ent* cpu, int maxBasicLeaf, - int maxExtendedLeaf) +detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, + uint32 maxExtendedLeaf) { if (currentCPU == 0) { status_t result = B_UNSUPPORTED; From 26c3861891cefb2246827c895d81dbb1a701a648 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 21:08:22 +0200 Subject: [PATCH 007/273] x86[_64]: Fix some style issues --- src/system/kernel/arch/x86/arch_cpu.cpp | 27 +++++++++++-------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 84b37f2e05..090e7fc87d 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -124,7 +124,7 @@ x86_optimized_functions gOptimizedFunctions = { }; /* CPU topology information */ -static uint32 (*getCPUTopologyID)(int currentCPU) = NULL; +static uint32 (*sGetCPUTopologyID)(int currentCPU); static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; @@ -559,10 +559,8 @@ computeCPUHierarchyMasks(int maxLogicalID, int maxCoreID) static uint32 -getCPULegacyInitialAPICID(int currentCPU) +getCPULegacyInitialAPICID(int /* currentCPU */) { - (void)currentCPU; - cpuid_info cpuid; get_current_cpuid(&cpuid, 1, 0); return cpuid.regs.ebx >> 24; @@ -572,7 +570,7 @@ getCPULegacyInitialAPICID(int currentCPU) static inline status_t detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) { - getCPUTopologyID = getCPULegacyInitialAPICID; + sGetCPUTopologyID = getCPULegacyInitialAPICID; cpuid_info cpuid; get_current_cpuid(&cpuid, 1, 0); @@ -617,8 +615,7 @@ detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) if (maxBasicLeaf < 11) return B_UNSUPPORTED; - const int kLevelCount = CPU_TOPOLOGY_LEVELS; - uint8 hierarchyLevels[kLevelCount]; + uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS]; int currentLevel = 0; int levelType; @@ -643,13 +640,13 @@ detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) } currentLevel++; - } while(levelType != 0 && levelsSet != 3); + } while (levelType != 0 && levelsSet != 3); - getCPUTopologyID = getIntelCPUInitialx2APICID; + sGetCPUTopologyID = getIntelCPUInitialx2APICID; - for (int i = 0; i < kLevelCount; i++) { + for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { uint32 mask = ~uint32(0); - if (i < kLevelCount - 1) + if (i < CPU_TOPOLOGY_LEVELS - 1) mask = (1 << hierarchyLevels[i]) - 1; if (i > 0) mask &= ~sHierarchyMask[i - 1]; @@ -664,7 +661,7 @@ detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) static inline status_t detectIntelCPUTopologyLegacy(uint32 maxBasicLeaf) { - getCPUTopologyID = getCPULegacyInitialAPICID; + sGetCPUTopologyID = getCPULegacyInitialAPICID; cpuid_info cpuid; @@ -717,7 +714,7 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, if (result != B_OK) { dprintf("No CPU topology information available.\n"); - getCPUTopologyID = getSimpleCPUTopologyID; + sGetCPUTopologyID = getSimpleCPUTopologyID; memset(sHierarchyMask, 0, sizeof(sHierarchyMask)); sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); @@ -726,8 +723,8 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, } } - ASSERT(getCPUTopologyID != NULL); - int topologyID = getCPUTopologyID(currentCPU); + ASSERT(sGetCPUTopologyID != NULL); + int topologyID = sGetCPUTopologyID(currentCPU); cpu->topology_id[CPU_TOPOLOGY_SMT] = getTopologyLevelID(topologyID, CPU_TOPOLOGY_SMT); cpu->topology_id[CPU_TOPOLOGY_CORE] From 1f50d09018c8cd7e573fcbdd8741712c5806ab7d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 21:24:46 +0200 Subject: [PATCH 008/273] kernel/util: Add bit hack utilities --- headers/private/kernel/util/BitUtils.h | 42 +++++++++++++++++++++++++ src/system/kernel/arch/x86/arch_cpu.cpp | 27 +--------------- 2 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 headers/private/kernel/util/BitUtils.h diff --git a/headers/private/kernel/util/BitUtils.h b/headers/private/kernel/util/BitUtils.h new file mode 100644 index 0000000000..60cbcb0a0a --- /dev/null +++ b/headers/private/kernel/util/BitUtils.h @@ -0,0 +1,42 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ +#ifndef KERNEL_UTIL_BITUTIL_H +#define KERNEL_UTIL_BITUTIL_H + + +#include + + +// http://graphics.stanford.edu/~seander/bithacks.html +static inline uint32 +nextPowerOf2(uint32 v) +{ + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + + return v; +} + + +// http://graphics.stanford.edu/~seander/bithacks.html +static inline uint32 +countSetBits(uint32 v) +{ + v = v - ((v >> 1) & 0x55555555); + v = (v & 0x33333333) + ((v >> 2) & 0x33333333); + return (((v + (v >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24; +} + + +#endif // KERNEL_UTIL_RANDOM_H + diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 090e7fc87d..da9bc039c5 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -129,32 +130,6 @@ static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; -// http://graphics.stanford.edu/~seander/bithacks.html -static inline uint32 -nextPowerOf2(uint32 v) -{ - v--; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - v++; - - return v; -} - - -// http://graphics.stanford.edu/~seander/bithacks.html -static inline uint32 -countSetBits(uint32 v) -{ - v = v - ((v >> 1) & 0x55555555); - v = (v & 0x33333333) + ((v >> 2) & 0x33333333); - return (((v + (v >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24; -} - - static status_t acpi_shutdown(bool rebootSystem) { From 36cc64a9b3f1744e7a030248bb81526e9f37f3d6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 2 Oct 2013 23:48:03 +0200 Subject: [PATCH 009/273] x86[_64]: Add CPU cache topology detection for AMD and Intel CPUs --- headers/private/kernel/arch/x86/arch_cpu.h | 8 ++ headers/private/kernel/cpu.h | 1 + src/system/kernel/arch/x86/arch_cpu.cpp | 117 ++++++++++++++++++++- 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 4809528c57..c51ee6f2fd 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -24,6 +24,9 @@ #endif // !_ASSEMBLER +#define CPU_MAX_CACHE_LEVEL 8 + + // MSR registers (possibly Intel specific) #define IA32_MSR_TSC 0x10 #define IA32_MSR_APIC_BASE 0x1b @@ -152,6 +155,10 @@ #define IA32_FEATURE_EXT_RDRND (1 << 30) // RDRAND instruction #define IA32_FEATURE_EXT_HYPERVISOR (1 << 31) // Running on a hypervisor +// x86 features from cpuid eax 0x80000001, ecx register (AMD) +#define IA32_FEATURE_AMD_EXT_CMPLEGACY (1 << 1) // Core MP legacy mode +#define IA32_FEATURE_AMD_EXT_TOPOLOGY (1 << 22) // Topology extensions + // x86 features from cpuid eax 0x80000001, edx register (AMD) // only care about the ones that are unique to this register #define IA32_FEATURE_AMD_EXT_SYSCALL (1 << 11) // SYSCALL/SYSRET @@ -265,6 +272,7 @@ typedef struct x86_cpu_module_info { enum x86_feature_type { FEATURE_COMMON = 0, // cpuid eax=1, ecx register FEATURE_EXT, // cpuid eax=1, edx register + FEATURE_EXT_AMD_ECX, // cpuid eax=0x80000001, ecx register (AMD) FEATURE_EXT_AMD, // cpuid eax=0x80000001, edx register (AMD) FEATURE_6_EAX, // cpuid eax=6, eax registers FEATURE_6_ECX, // cpuid eax=6, ecx registers diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index ab2a850d03..2432cce524 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -67,6 +67,7 @@ typedef struct cpu_ent { // CPU topology information int topology_id[CPU_TOPOLOGY_LEVELS]; + int cache_id[CPU_MAX_CACHE_LEVEL]; // arch-specific stuff arch_cpu_info arch; diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index da9bc039c5..f1b2d553b5 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -129,6 +129,10 @@ static uint32 (*sGetCPUTopologyID)(int currentCPU); static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS]; static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; +/* Cache topology information */ +static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; +static uint32 sCacheLevelCount; + static status_t acpi_shutdown(bool rebootSystem) @@ -563,7 +567,8 @@ detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) if (maxExtendedLeaf >= 0x80000001) { get_current_cpuid(&cpuid, 0x80000001, 0); - if ((cpuid.regs.ecx & 2) != 0) + if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY, + FEATURE_EXT_AMD_ECX)) maxCoreID = maxLogicalID; } @@ -573,11 +578,46 @@ detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) } -static uint32 -getIntelCPUInitialx2APICID(int currentCPU) +static void +detectAMDCacheTopology(uint32 maxExtendedLeaf) { - (void)currentCPU; + if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) + return; + if (maxExtendedLeaf < 0x8000001d) + return; + + uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; + uint8 maxCacheLevel = 0; + + int currentLevel = 0; + int cacheType; + do { + cpuid_info cpuid; + get_current_cpuid(&cpuid, 0x8000001d, currentLevel); + + cacheType = cpuid.regs.eax & 0x1f; + int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; + + int coresCount = nextPowerOf2(((cpuid.regs.eax >> 14) & 0x3f) + 1); + hierarchyLevels[cacheLevel - 1] + = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); + + if (cacheType != 0) + maxCacheLevel = max_c(maxCacheLevel, cacheLevel); + + currentLevel++; + } while (cacheType != 0); + + for (int i = 0; i < maxCacheLevel; i++) + sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); + sCacheLevelCount = maxCacheLevel; +} + + +static uint32 +getIntelCPUInitialx2APICID(int /* currentCPU */) +{ cpuid_info cpuid; get_current_cpuid(&cpuid, 11, 0); return cpuid.regs.edx; @@ -655,6 +695,39 @@ detectIntelCPUTopologyLegacy(uint32 maxBasicLeaf) } +static void +detectIntelCacheTopology(uint32 maxBasicLeaf) +{ + if (maxBasicLeaf < 4) + return; + + uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; + uint8 maxCacheLevel = 0; + + int currentLevel = 0; + int cacheType; + do { + cpuid_info cpuid; + get_current_cpuid(&cpuid, 4, currentLevel); + + cacheType = cpuid.regs.eax & 0x1f; + int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; + + hierarchyLevels[cacheLevel - 1] + = nextPowerOf2(((cpuid.regs.eax >> 14) & 0x3f) + 1); + + if (cacheType != 0) + maxCacheLevel = max_c(maxCacheLevel, cacheLevel); + + currentLevel++; + } while (cacheType != 0); + + for (int i = 0; i < maxCacheLevel; i++) + sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); + sCacheLevelCount = maxCacheLevel; +} + + static uint32 getSimpleCPUTopologyID(int currentCPU) { @@ -675,14 +748,24 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, uint32 maxExtendedLeaf) { if (currentCPU == 0) { + memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask)); + status_t result = B_UNSUPPORTED; if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { - if (cpu->arch.vendor == VENDOR_AMD) + if (cpu->arch.vendor == VENDOR_AMD) { result = detectAMDCPUTopology(maxBasicLeaf, maxExtendedLeaf); + + if (result == B_OK) + detectAMDCacheTopology(maxExtendedLeaf); + } + if (cpu->arch.vendor == VENDOR_INTEL) { result = detectIntelCPUTopologyx2APIC(maxBasicLeaf); if (result != B_OK) result = detectIntelCPUTopologyLegacy(maxBasicLeaf); + + if (result == B_OK) + detectIntelCacheTopology(maxBasicLeaf); } } @@ -707,11 +790,33 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, cpu->topology_id[CPU_TOPOLOGY_PACKAGE] = getTopologyLevelID(topologyID, CPU_TOPOLOGY_PACKAGE); + int i; + for (i = 0; i < sCacheLevelCount; i++) + cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; + for (; i < CPU_MAX_CACHE_LEVEL; i++) + cpu->cache_id[i] = -1; + #if DUMP_CPU_TOPOLOGY dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU, topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE], cpu->topology_id[CPU_TOPOLOGY_CORE], cpu->topology_id[CPU_TOPOLOGY_SMT]); + + if (sCacheLevelCount > 0) { + char cacheLevels[256]; + int offset = 0; + for (int i = 0; i < sCacheLevelCount; i++) { + offset += snprintf(cacheLevels + offset, + sizeof(cacheLevels) - offset, + " L%d id %d%s", i + 1, cpu->cache_id[i], + i < sCacheLevelCount - 1 ? "," : ""); + + if (offset >= sizeof(cacheLevels)) + break; + } + + dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels); + } #endif } @@ -820,6 +925,8 @@ detect_cpu(int currentCPU) if (maxExtendedLeaf >= 0x80000001) { get_current_cpuid(&cpuid, 0x80000001, 0); + if (cpu->arch.vendor == VENDOR_AMD) + cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx if (cpu->arch.vendor != VENDOR_AMD) cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; From 7087b865e20218d675add23731e96286a4cf2e0c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 3 Oct 2013 04:26:21 +0200 Subject: [PATCH 010/273] x86[_64]: Remove superfluous memset()s --- src/system/kernel/arch/x86/arch_cpu.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index f1b2d553b5..2a4365c989 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -724,6 +724,7 @@ detectIntelCacheTopology(uint32 maxBasicLeaf) for (int i = 0; i < maxCacheLevel; i++) sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); + sCacheLevelCount = maxCacheLevel; } @@ -774,10 +775,7 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, sGetCPUTopologyID = getSimpleCPUTopologyID; - memset(sHierarchyMask, 0, sizeof(sHierarchyMask)); sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); - - memset(sHierarchyShift, 0, sizeof(sHierarchyShift)); } } From 149c82a8ec8b1faed2ae722852042ba9d603b552 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 3 Oct 2013 04:27:49 +0200 Subject: [PATCH 011/273] kernel/util: Add bitmap implementation --- headers/private/kernel/util/BitUtils.h | 18 ++++++ headers/private/kernel/util/Bitmap.h | 81 ++++++++++++++++++++++++++ src/system/kernel/util/Bitmap.cpp | 68 +++++++++++++++++++++ src/system/kernel/util/Jamfile | 1 + 4 files changed, 168 insertions(+) create mode 100644 headers/private/kernel/util/Bitmap.h create mode 100644 src/system/kernel/util/Bitmap.cpp diff --git a/headers/private/kernel/util/BitUtils.h b/headers/private/kernel/util/BitUtils.h index 60cbcb0a0a..1a0e08dc61 100644 --- a/headers/private/kernel/util/BitUtils.h +++ b/headers/private/kernel/util/BitUtils.h @@ -38,5 +38,23 @@ countSetBits(uint32 v) } +static inline uint32 +log2(uint32 v) +{ + static const int MultiplyDeBruijnBitPosition[32] = { + 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 + }; + + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + + return MultiplyDeBruijnBitPosition[(uint32)(v * 0x07C4ACDDU) >> 27]; +} + + #endif // KERNEL_UTIL_RANDOM_H diff --git a/headers/private/kernel/util/Bitmap.h b/headers/private/kernel/util/Bitmap.h new file mode 100644 index 0000000000..b47854cc61 --- /dev/null +++ b/headers/private/kernel/util/Bitmap.h @@ -0,0 +1,81 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ +#ifndef KERNEL_UTIL_BITMAP_H +#define KERNEL_UTIL_BITMAP_H + + +#include +#include + + +class Bitmap { +public: + Bitmap(int bitCount); + ~Bitmap(); + + inline status_t GetInitStatus(); + + inline bool Get(int index) const; + inline void Set(int index); + inline void Clear(int index); + + int GetHighestSet() const; + +private: + status_t fInitStatus; + + int fElementsCount; + int fSize; + addr_t* fBits; + + static const int kBitsPerElement; +}; + + +status_t +Bitmap::GetInitStatus() +{ + return fInitStatus; +} + + +bool +Bitmap::Get(int index) const +{ + ASSERT(index < fSize); + + const int kArrayElement = index / kBitsPerElement; + const addr_t kBitMask = addr_t(1) << (index % kBitsPerElement); + return fBits[kArrayElement] & kBitMask; +} + + +void +Bitmap::Set(int index) +{ + ASSERT(index < fSize); + + const int kArrayElement = index / kBitsPerElement; + const addr_t kBitMask = addr_t(1) << (index % kBitsPerElement); + fBits[kArrayElement] |= kBitMask; +} + + +void +Bitmap::Clear(int index) +{ + ASSERT(index < fSize); + + const int kArrayElement = index / kBitsPerElement; + const addr_t kBitMask = addr_t(1) << (index % kBitsPerElement); + fBits[kArrayElement] &= ~addr_t(kBitMask); +} + + +#endif // KERNEL_UTIL_BITMAP_H + diff --git a/src/system/kernel/util/Bitmap.cpp b/src/system/kernel/util/Bitmap.cpp new file mode 100644 index 0000000000..588e996006 --- /dev/null +++ b/src/system/kernel/util/Bitmap.cpp @@ -0,0 +1,68 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ + + +#include + +#include + +#include + +#include + + +const int Bitmap::kBitsPerElement = sizeof(addr_t) * 8; + + +Bitmap::Bitmap(int bitCount) + : + fInitStatus(B_OK), + fElementsCount(0), + fSize(bitCount) +{ + int count = fSize + kBitsPerElement - 1; + count /= kBitsPerElement; + + fBits = new(std::nothrow) addr_t[count]; + if (fBits == NULL) { + fSize = 0; + fInitStatus = B_NO_MEMORY; + } + + fElementsCount = count; + memset(fBits, 0, sizeof(addr_t) * count); +} + + +Bitmap::~Bitmap() +{ + delete[] fBits; +} + + +int +Bitmap::GetHighestSet() const +{ + int i = fElementsCount - 1; + while (i >= 0 && fBits[i] == 0) + i--; + + if (i < 0) + return -1; + + STATIC_ASSERT(sizeof(addr_t) == sizeof(uint64) + || sizeof(addr_t) == sizeof(uint32)); + if (sizeof(addr_t) == sizeof(uint32)) + return log2(fBits[i]) + i * kBitsPerElement; + + uint32 v = fBits[i] >> 32; + if (v != 0) + return log2(v) + sizeof(uint32) * 8 + i * kBitsPerElement; + return log2(fBits[i]) + i * kBitsPerElement; +} + diff --git a/src/system/kernel/util/Jamfile b/src/system/kernel/util/Jamfile index a8f49b9d2a..6b89e4121c 100644 --- a/src/system/kernel/util/Jamfile +++ b/src/system/kernel/util/Jamfile @@ -4,6 +4,7 @@ UsePrivateHeaders [ FDirName kernel util ] ; KernelMergeObject kernel_util.o : AVLTreeBase.cpp + Bitmap.cpp hostname.cpp inet_addr.c inet_ntop.c From 7039b950fb70c6138b7e75f1ec7f8df7eaf0740c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 5 Oct 2013 18:03:00 +0200 Subject: [PATCH 012/273] x86[_64]: Fix style issues --- headers/private/kernel/util/BitUtils.h | 4 +- src/system/kernel/arch/x86/arch_cpu.cpp | 74 ++++++++++++------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/headers/private/kernel/util/BitUtils.h b/headers/private/kernel/util/BitUtils.h index 1a0e08dc61..55191d0079 100644 --- a/headers/private/kernel/util/BitUtils.h +++ b/headers/private/kernel/util/BitUtils.h @@ -14,7 +14,7 @@ // http://graphics.stanford.edu/~seander/bithacks.html static inline uint32 -nextPowerOf2(uint32 v) +next_power_of_2(uint32 v) { v--; v |= v >> 1; @@ -30,7 +30,7 @@ nextPowerOf2(uint32 v) // http://graphics.stanford.edu/~seander/bithacks.html static inline uint32 -countSetBits(uint32 v) +count_set_bits(uint32 v) { v = v - ((v >> 1) & 0x55555555); v = (v & 0x33333333) + ((v >> 2) & 0x33333333); diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 2a4365c989..ed1e7ebb82 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -518,7 +518,7 @@ dump_feature_string(int currentCPU, cpu_ent* cpu) static void -computeCPUHierarchyMasks(int maxLogicalID, int maxCoreID) +compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID) { ASSERT(maxLogicalID >= maxCoreID); const int kMaxSMTID = maxLogicalID / maxCoreID; @@ -528,17 +528,17 @@ computeCPUHierarchyMasks(int maxLogicalID, int maxCoreID) sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID; sHierarchyShift[CPU_TOPOLOGY_CORE] - = countSetBits(sHierarchyMask[CPU_TOPOLOGY_SMT]); + = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]); const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT] | sHierarchyMask[CPU_TOPOLOGY_CORE]; sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask; - sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = countSetBits(kSinglePackageMask); + sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask); } static uint32 -getCPULegacyInitialAPICID(int /* currentCPU */) +get_cpu_legacy_initial_apic_id(int /* currentCPU */) { cpuid_info cpuid; get_current_cpuid(&cpuid, 1, 0); @@ -547,13 +547,13 @@ getCPULegacyInitialAPICID(int /* currentCPU */) static inline status_t -detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) +detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) { - sGetCPUTopologyID = getCPULegacyInitialAPICID; + sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; cpuid_info cpuid; get_current_cpuid(&cpuid, 1, 0); - int maxLogicalID = nextPowerOf2((cpuid.regs.ebx >> 16) & 0xff); + int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); int maxCoreID = 1; if (maxExtendedLeaf >= 0x80000008) { @@ -562,7 +562,7 @@ detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) if (maxCoreID != 0) maxCoreID = 1 << maxCoreID; else - maxCoreID = nextPowerOf2((cpuid.regs.edx & 0xf) + 1); + maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1); } if (maxExtendedLeaf >= 0x80000001) { @@ -572,14 +572,14 @@ detectAMDCPUTopology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf) maxCoreID = maxLogicalID; } - computeCPUHierarchyMasks(maxLogicalID, maxCoreID); + compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); return B_OK; } static void -detectAMDCacheTopology(uint32 maxExtendedLeaf) +detect_amd_cache_topology(uint32 maxExtendedLeaf) { if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX)) return; @@ -599,7 +599,7 @@ detectAMDCacheTopology(uint32 maxExtendedLeaf) cacheType = cpuid.regs.eax & 0x1f; int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; - int coresCount = nextPowerOf2(((cpuid.regs.eax >> 14) & 0x3f) + 1); + int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); hierarchyLevels[cacheLevel - 1] = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); @@ -616,7 +616,7 @@ detectAMDCacheTopology(uint32 maxExtendedLeaf) static uint32 -getIntelCPUInitialx2APICID(int /* currentCPU */) +get_intel_cpu_initial_x2apic_id(int /* currentCPU */) { cpuid_info cpuid; get_current_cpuid(&cpuid, 11, 0); @@ -625,7 +625,7 @@ getIntelCPUInitialx2APICID(int /* currentCPU */) static inline status_t -detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) +detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf) { if (maxBasicLeaf < 11) return B_UNSUPPORTED; @@ -657,7 +657,7 @@ detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) currentLevel++; } while (levelType != 0 && levelsSet != 3); - sGetCPUTopologyID = getIntelCPUInitialx2APICID; + sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id; for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) { uint32 mask = ~uint32(0); @@ -674,29 +674,29 @@ detectIntelCPUTopologyx2APIC(uint32 maxBasicLeaf) static inline status_t -detectIntelCPUTopologyLegacy(uint32 maxBasicLeaf) +detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf) { - sGetCPUTopologyID = getCPULegacyInitialAPICID; + sGetCPUTopologyID = get_cpu_legacy_initial_apic_id; cpuid_info cpuid; get_current_cpuid(&cpuid, 1, 0); - int maxLogicalID = nextPowerOf2((cpuid.regs.ebx >> 16) & 0xff); + int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff); int maxCoreID = 1; if (maxBasicLeaf >= 4) { get_current_cpuid(&cpuid, 4, 0); - maxCoreID = nextPowerOf2((cpuid.regs.eax >> 26) + 1); + maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1); } - computeCPUHierarchyMasks(maxLogicalID, maxCoreID); + compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID); return B_OK; } static void -detectIntelCacheTopology(uint32 maxBasicLeaf) +detect_intel_cache_topology(uint32 maxBasicLeaf) { if (maxBasicLeaf < 4) return; @@ -714,7 +714,7 @@ detectIntelCacheTopology(uint32 maxBasicLeaf) int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; hierarchyLevels[cacheLevel - 1] - = nextPowerOf2(((cpuid.regs.eax >> 14) & 0x3f) + 1); + = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); if (cacheType != 0) maxCacheLevel = max_c(maxCacheLevel, cacheLevel); @@ -730,14 +730,14 @@ detectIntelCacheTopology(uint32 maxBasicLeaf) static uint32 -getSimpleCPUTopologyID(int currentCPU) +get_simple_cpu_topology_id(int currentCPU) { return currentCPU; } static inline int -getTopologyLevelID(uint32 id, cpu_topology_level level) +get_topology_level_id(uint32 id, cpu_topology_level level) { ASSERT(level < CPU_TOPOLOGY_LEVELS); return (id & sHierarchyMask[level]) >> sHierarchyShift[level]; @@ -745,7 +745,7 @@ getTopologyLevelID(uint32 id, cpu_topology_level level) static void -detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, +detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, uint32 maxExtendedLeaf) { if (currentCPU == 0) { @@ -754,26 +754,26 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, status_t result = B_UNSUPPORTED; if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) { if (cpu->arch.vendor == VENDOR_AMD) { - result = detectAMDCPUTopology(maxBasicLeaf, maxExtendedLeaf); + result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf); if (result == B_OK) - detectAMDCacheTopology(maxExtendedLeaf); + detect_amd_cache_topology(maxExtendedLeaf); } if (cpu->arch.vendor == VENDOR_INTEL) { - result = detectIntelCPUTopologyx2APIC(maxBasicLeaf); + result = detect_intel_cpu_topology_x2apic(maxBasicLeaf); if (result != B_OK) - result = detectIntelCPUTopologyLegacy(maxBasicLeaf); + result = detect_intel_cpu_topology_legacy(maxBasicLeaf); if (result == B_OK) - detectIntelCacheTopology(maxBasicLeaf); + detect_intel_cache_topology(maxBasicLeaf); } } if (result != B_OK) { dprintf("No CPU topology information available.\n"); - sGetCPUTopologyID = getSimpleCPUTopologyID; + sGetCPUTopologyID = get_simple_cpu_topology_id; sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0); } @@ -782,13 +782,13 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, ASSERT(sGetCPUTopologyID != NULL); int topologyID = sGetCPUTopologyID(currentCPU); cpu->topology_id[CPU_TOPOLOGY_SMT] - = getTopologyLevelID(topologyID, CPU_TOPOLOGY_SMT); + = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT); cpu->topology_id[CPU_TOPOLOGY_CORE] - = getTopologyLevelID(topologyID, CPU_TOPOLOGY_CORE); + = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE); cpu->topology_id[CPU_TOPOLOGY_PACKAGE] - = getTopologyLevelID(topologyID, CPU_TOPOLOGY_PACKAGE); + = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); - int i; + unsigned int i; for (i = 0; i < sCacheLevelCount; i++) cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; for (; i < CPU_MAX_CACHE_LEVEL; i++) @@ -802,8 +802,8 @@ detectCPUTopology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, if (sCacheLevelCount > 0) { char cacheLevels[256]; - int offset = 0; - for (int i = 0; i < sCacheLevelCount; i++) { + unsigned int offset = 0; + for (i = 0; i < sCacheLevelCount; i++) { offset += snprintf(cacheLevels + offset, sizeof(cacheLevels) - offset, " L%d id %d%s", i + 1, cpu->cache_id[i], @@ -936,7 +936,7 @@ detect_cpu(int currentCPU) cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; } - detectCPUTopology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); + detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); #if DUMP_FEATURE_STRING dump_feature_string(currentCPU, cpu); From b8c1df9b004a2602f23d0ad046efbe7924b62847 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 5 Oct 2013 20:16:06 +0200 Subject: [PATCH 013/273] kernel: Add O(1) lookup and insertion priority queue --- headers/private/kernel/RunQueueLink.h | 56 ++++ headers/private/kernel/thread_types.h | 4 +- src/system/kernel/scheduler/RunQueue.h | 353 +++++++++++++++++++++++++ 3 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 headers/private/kernel/RunQueueLink.h create mode 100644 src/system/kernel/scheduler/RunQueue.h diff --git a/headers/private/kernel/RunQueueLink.h b/headers/private/kernel/RunQueueLink.h new file mode 100644 index 0000000000..79fdc60a1f --- /dev/null +++ b/headers/private/kernel/RunQueueLink.h @@ -0,0 +1,56 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ +#ifndef RUN_QUEUE_LINK_H +#define RUN_QUEUE_LINK_H + + +template +struct RunQueueLink { + RunQueueLink(); + + unsigned int fPriority; + Element* fPrevious; + Element* fNext; +}; + +template +class RunQueueLinkImpl { +public: + inline RunQueueLink* GetRunQueueLink(); + +private: + RunQueueLink fRunQueueLink; +}; + + +#if KDEBUG +template +RunQueueLink::RunQueueLink() + : + fPrevious(NULL), + fNext(NULL) +{ +} +#else +template +RunQueueLink::RunQueueLink() +{ +} +#endif + + +template +RunQueueLink* +RunQueueLinkImpl::GetRunQueueLink() +{ + return &fRunQueueLink; +} + + +#endif // RUN_QUEUE_LINK_H + diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 1c06271a50..637c5bed42 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -409,7 +410,8 @@ private: }; -struct Thread : TeamThreadIteratorEntry, KernelReferenceable { +struct Thread : TeamThreadIteratorEntry, KernelReferenceable, + RunQueueLinkImpl { int32 flags; // summary of events relevant in interrupt // handlers (signals pending, user debugging // enabled, etc.) diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h new file mode 100644 index 0000000000..9485a1c4f8 --- /dev/null +++ b/src/system/kernel/scheduler/RunQueue.h @@ -0,0 +1,353 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ +#ifndef RUN_QUEUE_H +#define RUN_QUEUE_H + + +#include +#include + + +template +class RunQueueStandardGetLink { +private: + typedef RunQueueLink Link; + +public: + inline Link* operator()(Element* element) const; +}; + +template Element::*LinkMember> +class RunQueueMemberGetLink { +private: + typedef RunQueueLink Link; + +public: + inline Link* operator()(Element* element) const; +}; + +#define RUN_QUEUE_TEMPLATE_LIST \ + template +#define RUN_QUEUE_CLASS_NAME RunQueue + +template > +class RunQueue { +public: + class ConstIterator { + public: + ConstIterator(); + ConstIterator(const RunQueue* list); + + inline ConstIterator& operator=(const ConstIterator& other); + + bool HasNext() const; + Element* Next(); + + void Rewind(); + + private: + inline void _FindNextPriority(); + + const RUN_QUEUE_CLASS_NAME* fList; + unsigned int fPriority; + Element* fNext; + + static GetLink sGetLink; + }; + + RunQueue(); + + inline status_t GetInitStatus(); + + inline Element* PeekMaximum(); + inline Element* PeekSecondMaximum(); + + inline void PushFront(Element* element, unsigned int priority); + inline void PushBack(Element* elementt, unsigned int priority); + + inline void Remove(Element* element); + + inline Element* GetHead(unsigned int priority) const; + + inline ConstIterator GetConstIterator() const; + +private: + status_t fInitStatus; + + Bitmap fBitmap; + + Element* fHeads[MaxPriority + 1]; + Element* fTails[MaxPriority + 1]; + + static GetLink sGetLink; +}; + + +template +RunQueueLink* +RunQueueStandardGetLink::operator()(Element* element) const +{ + return element->GetRunQueueLink(); +} + + +template Element::*LinkMember> +RunQueueLink* +RunQueueMemberGetLink::operator()(Element* element) const +{ + return &(element->*LinkMember); +} + + +RUN_QUEUE_TEMPLATE_LIST +RUN_QUEUE_CLASS_NAME::ConstIterator::ConstIterator() + : + fList(NULL) +{ +} + + +RUN_QUEUE_TEMPLATE_LIST +RUN_QUEUE_CLASS_NAME::ConstIterator::ConstIterator(const RunQueue* list) + : + fList(list) +{ + Rewind(); +} + + +RUN_QUEUE_TEMPLATE_LIST +typename RUN_QUEUE_CLASS_NAME::ConstIterator& +RUN_QUEUE_CLASS_NAME::ConstIterator::operator=(const ConstIterator& other) +{ + fList = other.fList; + fPriority = other.fPriority; + fNext = other.fNext; + + return *this; +} + + +RUN_QUEUE_TEMPLATE_LIST +bool +RUN_QUEUE_CLASS_NAME::ConstIterator::HasNext() const +{ + return fNext != NULL; +} + + +RUN_QUEUE_TEMPLATE_LIST +Element* +RUN_QUEUE_CLASS_NAME::ConstIterator::Next() +{ + ASSERT(HasNext()); + + Element* current = fNext; + RunQueueLink* link = sGetLink(fNext); + + fNext = link->fNext; + if (fNext == NULL) + _FindNextPriority(); + + return current; +} + + +RUN_QUEUE_TEMPLATE_LIST +void +RUN_QUEUE_CLASS_NAME::ConstIterator::Rewind() +{ + ASSERT(fList != NULL); + + fPriority = MaxPriority; + fNext = fList->GetHead(fPriority); + if (fNext == NULL) + _FindNextPriority(); +} + + +RUN_QUEUE_TEMPLATE_LIST +void +RUN_QUEUE_CLASS_NAME::ConstIterator::_FindNextPriority() +{ + ASSERT(fList != NULL); + + while (fPriority-- > 0) { + fNext = fList->GetHead(fPriority); + if (fNext != NULL) + break; + } +} + + +RUN_QUEUE_TEMPLATE_LIST +RUN_QUEUE_CLASS_NAME::RunQueue() + : + fBitmap(MaxPriority + 1) +{ + fInitStatus = fBitmap.GetInitStatus(); + + memset(fHeads, 0, sizeof(fHeads)); + memset(fTails, 0, sizeof(fTails)); +} + + +RUN_QUEUE_TEMPLATE_LIST +status_t +RUN_QUEUE_CLASS_NAME::GetInitStatus() +{ + return fInitStatus; +} + + +RUN_QUEUE_TEMPLATE_LIST +Element* +RUN_QUEUE_CLASS_NAME::PeekMaximum() +{ + int priority = fBitmap.GetHighestSet(); + if (priority < 0) + return NULL; + + ASSERT((unsigned int)priority <= MaxPriority); + ASSERT(fHeads[priority] != NULL); + + Element* element = fHeads[priority]; + RunQueueLink* elementLink = sGetLink(element); + + ASSERT(elementLink->fPriority == (unsigned int)priority); + ASSERT(fTails[priority] != NULL); + ASSERT(elementLink->fPrevious == NULL); + + return element; +} + + +RUN_QUEUE_TEMPLATE_LIST +Element* +RUN_QUEUE_CLASS_NAME::PeekSecondMaximum() +{ + int priority = fBitmap.GetHighestSet(); + if (priority < 0) + return NULL; + + fBitmap.Clear(priority); + Element* element = PeekMaximum(); + fBitmap.Set(priority); + + return element; +} + + +RUN_QUEUE_TEMPLATE_LIST +void +RUN_QUEUE_CLASS_NAME::PushFront(Element* element, + unsigned int priority) +{ + ASSERT(priority <= MaxPriority); + + RunQueueLink* elementLink = sGetLink(element); + + ASSERT(elementLink->fPrevious == NULL); + ASSERT(elementLink->fNext == NULL); + + ASSERT((fHeads[priority] == NULL && fTails[priority] == NULL) + || (fHeads[priority] != NULL && fTails[priority] != NULL)); + + elementLink->fPriority = priority; + elementLink->fNext = fHeads[priority]; + if (fHeads[priority]) + sGetLink(fHeads[priority])->fPrevious = element; + else + fTails[priority] = element; + fHeads[priority] = element; + + fBitmap.Set(priority); +} + + +RUN_QUEUE_TEMPLATE_LIST +void +RUN_QUEUE_CLASS_NAME::PushBack(Element* element, + unsigned int priority) +{ + ASSERT(priority <= MaxPriority); + + RunQueueLink* elementLink = sGetLink(element); + + ASSERT(elementLink->fPrevious == NULL); + ASSERT(elementLink->fNext == NULL); + + ASSERT((fHeads[priority] == NULL && fTails[priority] == NULL) + || (fHeads[priority] != NULL && fTails[priority] != NULL)); + + elementLink->fPriority = priority; + elementLink->fPrevious = fTails[priority]; + if (fTails[priority]) + sGetLink(fTails[priority])->fNext = element; + else + fHeads[priority] = element; + fTails[priority] = element; + + fBitmap.Set(priority); +} + + +RUN_QUEUE_TEMPLATE_LIST +void +RUN_QUEUE_CLASS_NAME::Remove(Element* element) +{ + RunQueueLink* elementLink = sGetLink(element); + unsigned int priority = elementLink->fPriority; + + ASSERT(elementLink->fPrevious != NULL || fHeads[priority] == element); + ASSERT(elementLink->fNext != NULL || fTails[priority] == element); + + if (elementLink->fPrevious != NULL) + sGetLink(elementLink->fPrevious)->fNext = elementLink->fNext; + else + fHeads[priority] = elementLink->fNext; + if (elementLink->fNext != NULL) + sGetLink(elementLink->fNext)->fPrevious = elementLink->fPrevious; + else + fTails[priority] = elementLink->fPrevious; + + ASSERT((fHeads[priority] == NULL && fTails[priority] == NULL) + || (fHeads[priority] != NULL && fTails[priority] != NULL)); + + if (fHeads[priority] == NULL) + fBitmap.Clear(priority); + +#if KDEBUG + elementLink->fPrevious = NULL; + elementLink->fNext = NULL; +#endif +} + + +RUN_QUEUE_TEMPLATE_LIST +Element* +RUN_QUEUE_CLASS_NAME::GetHead(unsigned int priority) const +{ + ASSERT(priority <= MaxPriority); + return fHeads[priority]; +} + + +RUN_QUEUE_TEMPLATE_LIST +typename RUN_QUEUE_CLASS_NAME::ConstIterator +RUN_QUEUE_CLASS_NAME::GetConstIterator() const +{ + return ConstIterator(this); +} + + +#endif // RUN_QUEUE_H + From 9ad558f01c1a3f0f03a1646d0d44548d565aa60d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 5 Oct 2013 20:22:59 +0200 Subject: [PATCH 014/273] kernel: Use the new runqueue in non-MP scheduler --- .../kernel/scheduler/scheduler_simple.cpp | 132 ++++++------------ 1 file changed, 45 insertions(+), 87 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 6b60d3ca0c..399e65f173 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. * Copyright 2002, Angelo Mottola, a.mottola@libero.it. @@ -25,6 +26,7 @@ #include #include +#include "RunQueue.h" #include "scheduler_common.h" #include "scheduler_tracing.h" @@ -41,23 +43,23 @@ const bigtime_t kThreadQuantum = 3000; // The run queue. Holds the threads ready to run ordered by priority. -static Thread *sRunQueue = NULL; +static RunQueue* sRunQueue; static int dump_run_queue(int argc, char **argv) { - Thread *thread; + RunQueue::ConstIterator iterator; + iterator = sRunQueue->GetConstIterator(); - thread = sRunQueue; - if (!thread) + if (!iterator.HasNext()) kprintf("Run queue is empty!\n"); else { kprintf("thread id priority name\n"); - while (thread) { + while (iterator.HasNext()) { + Thread *thread = iterator.Next(); kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %s\n", thread, thread->id, thread->priority, thread->name); - thread = thread->queue_next; } } @@ -73,24 +75,9 @@ simple_enqueue_in_run_queue(Thread *thread) { thread->state = thread->next_state = B_THREAD_READY; - Thread *curr, *prev; - for (curr = sRunQueue, prev = NULL; curr - && curr->priority >= thread->next_priority; - curr = curr->queue_next) { - if (prev) - prev = prev->queue_next; - else - prev = sRunQueue; - } - - T(EnqueueThread(thread, prev, curr)); - - thread->queue_next = curr; - if (prev) - prev->queue_next = thread; - else - sRunQueue = thread; + //T(EnqueueThread(thread, prev, curr)); + sRunQueue->PushBack(thread, thread->next_priority); thread->next_priority = thread->priority; // notify listeners @@ -127,23 +114,8 @@ simple_set_thread_priority(Thread *thread, int32 priority) NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, thread); - // find thread in run queue - Thread *item, *prev; - for (item = sRunQueue, prev = NULL; item && item != thread; - item = item->queue_next) { - if (prev) - prev = prev->queue_next; - else - prev = sRunQueue; - } - - ASSERT(item == thread); - - // remove the thread - if (prev) - prev->queue_next = item->queue_next; - else - sRunQueue = item->queue_next; + // remove thread from run queue + sRunQueue->Remove(thread); // set priority and re-insert thread->priority = thread->next_priority = priority; @@ -185,8 +157,8 @@ reschedule_event(timer *unused) static void simple_reschedule(void) { - Thread *oldThread = thread_get_current_thread(); - Thread *nextThread, *prevThread; + Thread* oldThread = thread_get_current_thread(); + Thread* nextThread; // check whether we're only supposed to reschedule, if the current thread // is idle @@ -220,64 +192,48 @@ simple_reschedule(void) break; } - nextThread = sRunQueue; - prevThread = NULL; + while (true) { + // select thread with the biggest priority + nextThread = sRunQueue->PeekMaximum(); + if (!nextThread) + panic("reschedule(): run queue is empty!\n"); - while (nextThread) { - // select next thread from the run queue - while (nextThread && nextThread->priority > B_IDLE_PRIORITY) { #if 0 - if (oldThread == nextThread && nextThread->was_yielded) { - // ignore threads that called thread_yield() once - nextThread->was_yielded = false; - prevThread = nextThread; - nextThread = nextThread->queue_next; - } + if (oldThread == nextThread && nextThread->was_yielded) { + // ignore threads that called thread_yield() once + nextThread->was_yielded = false; + prevThread = nextThread; + nextThread = nextThread->queue_next; + } #endif - // always extract real time threads - if (nextThread->priority >= B_FIRST_REAL_TIME_PRIORITY) - break; + // always extract real time threads + if (nextThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + break; - // find next thread with lower priority - Thread *lowerNextThread = nextThread->queue_next; - Thread *lowerPrevThread = nextThread; - int32 priority = nextThread->priority; + // find thread with the second biggest priority + Thread* lowerNextThread = sRunQueue->PeekSecondMaximum(); - while (lowerNextThread != NULL - && priority == lowerNextThread->priority) { - lowerPrevThread = lowerNextThread; - lowerNextThread = lowerNextThread->queue_next; - } - // never skip last non-idle normal thread - if (lowerNextThread == NULL - || lowerNextThread->priority == B_IDLE_PRIORITY) - break; + // never skip last non-idle normal thread + if (lowerNextThread == NULL + || lowerNextThread->priority == B_IDLE_PRIORITY) + break; - int32 priorityDiff = priority - lowerNextThread->priority; - if (priorityDiff > 15) - break; + int32 priorityDiff = nextThread->priority - lowerNextThread->priority; + if (priorityDiff > 15) + break; - // skip normal threads sometimes - // (twice as probable per priority level) - if ((fast_random_value() >> (15 - priorityDiff)) != 0) - break; + // skip normal threads sometimes + // (twice as probable per priority level) + if ((fast_random_value() >> (15 - priorityDiff)) != 0) + break; - nextThread = lowerNextThread; - prevThread = lowerPrevThread; - } + nextThread = lowerNextThread; break; } - if (!nextThread) - panic("reschedule(): run queue is empty!\n"); - - // extract selected thread from the run queue - if (prevThread) - prevThread->queue_next = nextThread->queue_next; - else - sRunQueue = nextThread->queue_next; + sRunQueue->Remove(nextThread); T(ScheduleThread(nextThread, oldThread)); @@ -375,6 +331,8 @@ static scheduler_ops kSimpleOps = { void scheduler_simple_init() { + sRunQueue = new(std::nothrow) RunQueue; + gScheduler = &kSimpleOps; add_debugger_command_etc("run_queue", &dump_run_queue, From 6d7e2912332e6f40dda0724eff35d99ecdcc8371 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 5 Oct 2013 20:45:07 +0200 Subject: [PATCH 015/273] kernel: Allow scheduler initialization to fail --- src/system/kernel/scheduler/scheduler.cpp | 10 +++++++--- src/system/kernel/scheduler/scheduler_affine.cpp | 4 +++- src/system/kernel/scheduler/scheduler_affine.h | 2 +- src/system/kernel/scheduler/scheduler_simple.cpp | 12 +++++++++++- src/system/kernel/scheduler/scheduler_simple.h | 2 +- src/system/kernel/scheduler/scheduler_simple_smp.cpp | 4 +++- src/system/kernel/scheduler/scheduler_simple_smp.h | 2 +- 7 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index e4848d1d48..a8ed35a985 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -67,19 +67,23 @@ scheduler_init(void) dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s\n", cpuCount, cpuCount != 1 ? "s" : ""); + status_t result; if (cpuCount > 1) { #if 0 dprintf("scheduler_init: using affine scheduler\n"); - scheduler_affine_init(); + result = scheduler_affine_init(); #else dprintf("scheduler_init: using simple SMP scheduler\n"); - scheduler_simple_smp_init(); + result = scheduler_simple_smp_init(); #endif } else { dprintf("scheduler_init: using simple scheduler\n"); - scheduler_simple_init(); + result = scheduler_simple_init(); } + if (result != B_OK) + panic("scheduler_init: failed to initialize scheduler\n"); + // Disable rescheduling until the basic kernel initialization is done and // CPUs are ready to enable interrupts. sRescheduleFunction = gScheduler->reschedule; diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index c2a5c0c31d..078980ebf4 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -556,7 +556,7 @@ static scheduler_ops kAffineOps = { // #pragma mark - -void +status_t scheduler_affine_init() { gScheduler = &kAffineOps; @@ -564,4 +564,6 @@ scheduler_affine_init() memset(sRunQueueSize, 0, sizeof(sRunQueueSize)); add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); + + return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_affine.h b/src/system/kernel/scheduler/scheduler_affine.h index 1762fe37a7..3c52cd06fe 100644 --- a/src/system/kernel/scheduler/scheduler_affine.h +++ b/src/system/kernel/scheduler/scheduler_affine.h @@ -7,7 +7,7 @@ #define KERNEL_SCHEDULER_AFFINE_H -void scheduler_affine_init(); +status_t scheduler_affine_init(); #endif // KERNEL_SCHEDULER_AFFINE_H diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 399e65f173..606a1f936a 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -328,13 +328,23 @@ static scheduler_ops kSimpleOps = { // #pragma mark - -void +status_t scheduler_simple_init() { sRunQueue = new(std::nothrow) RunQueue; + if (sRunQueue == NULL) + return B_NO_MEMORY; + + status_t result = sRunQueue->GetInitStatus(); + if (result != B_OK) { + delete sRunQueue; + return result; + } gScheduler = &kSimpleOps; add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); + + return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_simple.h b/src/system/kernel/scheduler/scheduler_simple.h index bac69ac78c..aba7f5afbd 100644 --- a/src/system/kernel/scheduler/scheduler_simple.h +++ b/src/system/kernel/scheduler/scheduler_simple.h @@ -6,7 +6,7 @@ #define KERNEL_SCHEDULER_SIMPLE_H -void scheduler_simple_init(); +status_t scheduler_simple_init(); #endif // KERNEL_SCHEDULER_SIMPLE_H diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp index c4d1e8a2df..1a4a564353 100644 --- a/src/system/kernel/scheduler/scheduler_simple_smp.cpp +++ b/src/system/kernel/scheduler/scheduler_simple_smp.cpp @@ -471,7 +471,7 @@ static scheduler_ops kSimpleSMPOps = { // #pragma mark - -void +status_t scheduler_simple_smp_init() { sCPUCount = smp_get_num_cpus(); @@ -480,4 +480,6 @@ scheduler_simple_smp_init() add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); + + return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.h b/src/system/kernel/scheduler/scheduler_simple_smp.h index 67b482494c..d59a02841f 100644 --- a/src/system/kernel/scheduler/scheduler_simple_smp.h +++ b/src/system/kernel/scheduler/scheduler_simple_smp.h @@ -6,7 +6,7 @@ #define KERNEL_SCHEDULER_SIMPLE_SMP_H -void scheduler_simple_smp_init(); +status_t scheduler_simple_smp_init(); #endif // KERNEL_SCHEDULER_SIMPLE_SMP_H From 82c26e1f1fefa9b91861d23368723618bd7bdd12 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 7 Oct 2013 21:11:32 +0200 Subject: [PATCH 016/273] kernel: Punish CPU bound threads This patch appears to fix #8007. Thread that consume its whole quantum has its priority reduced. The penalty is cancelled when the thread voluntarily gives up CPU. Real-time threads are not affected. The problem of thread starvation is not solved completely. The worst case latency is still unbounded (even in systems with bounded number of threads). When a middle priority thread is constantly preempted by high priority threads it would not earn the penalty, thus the lower priority threads still can be starved. Moreover, the punishment is probably too aggressive as it reduces priority of virtually all CPU bound threads to 1. --- .../kernel/scheduler/scheduler_simple.cpp | 166 ++++++++++++------ 1 file changed, 114 insertions(+), 52 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 606a1f936a..6549dd42cc 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -46,6 +46,23 @@ const bigtime_t kThreadQuantum = 3000; static RunQueue* sRunQueue; +struct scheduler_thread_data { + scheduler_thread_data() { Init(); } + void Init(); + + int32 priority_penalty; + bool lost_cpu; +}; + + +void +scheduler_thread_data::Init() +{ + priority_penalty = 0; + lost_cpu = false; +} + + static int dump_run_queue(int argc, char **argv) { @@ -55,11 +72,15 @@ dump_run_queue(int argc, char **argv) if (!iterator.HasNext()) kprintf("Run queue is empty!\n"); else { - kprintf("thread id priority name\n"); + kprintf("thread id priority penalty name\n"); while (iterator.HasNext()) { Thread *thread = iterator.Next(); - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %s\n", thread, - thread->id, thread->priority, thread->name); + scheduler_thread_data* schedulerThreadData + = reinterpret_cast( + thread->scheduler_data); + kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", + thread, thread->id, thread->priority, + schedulerThreadData->priority_penalty, thread->name); } } @@ -67,6 +88,57 @@ dump_run_queue(int argc, char **argv) } +static inline int32 +simple_get_effective_priority(Thread *thread) +{ + if (thread->priority == B_IDLE_PRIORITY) + return thread->priority; + + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + + int32 effectivePriority = thread->priority; + if (effectivePriority < B_FIRST_REAL_TIME_PRIORITY) + effectivePriority -= schedulerThreadData->priority_penalty; + + ASSERT(schedulerThreadData->priority_penalty >= 0); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return min_c(effectivePriority, thread->next_priority); +} + + +static inline void +simple_increase_penalty(Thread *thread) +{ + if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) + return; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return; + + TRACE(("increasing thread %ld penalty\n", thread->id)); + + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + int32 oldPenalty = schedulerThreadData->priority_penalty++; + + ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); + if (thread->priority - oldPenalty <= B_LOWEST_ACTIVE_PRIORITY) + schedulerThreadData->priority_penalty = oldPenalty; +} + + +static inline void +simple_cancel_penalty(Thread *thread) +{ + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + if (schedulerThreadData->priority_penalty != 0) + TRACE(("cancelling thread %ld penalty\n", thread->id)); + schedulerThreadData->priority_penalty = 0; +} + + /*! Enqueues the thread into the run queue. Note: thread lock must be held when entering this function */ @@ -77,14 +149,21 @@ simple_enqueue_in_run_queue(Thread *thread) //T(EnqueueThread(thread, prev, curr)); - sRunQueue->PushBack(thread, thread->next_priority); + int32 threadPriority = simple_get_effective_priority(thread); + sRunQueue->PushBack(thread, threadPriority); thread->next_priority = thread->priority; // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - if (thread->priority > thread_get_current_thread()->priority) { + Thread* currentThread = thread_get_current_thread(); + if (threadPriority > currentThread->priority) { + scheduler_thread_data* schedulerThreadData + = reinterpret_cast( + currentThread->scheduler_data); + + schedulerThreadData->lost_cpu = true; gCPU[0].invoke_scheduler = true; gCPU[0].invoke_scheduler_if_idle = false; } @@ -118,7 +197,9 @@ simple_set_thread_priority(Thread *thread, int32 priority) sRunQueue->Remove(thread); // set priority and re-insert + simple_cancel_penalty(thread); thread->priority = thread->next_priority = priority; + simple_enqueue_in_run_queue(thread); } @@ -144,9 +225,14 @@ reschedule_event(timer *unused) { // This function is called as a result of the timer event set by the // scheduler. Make sure the reschedule() is invoked. - thread_get_current_thread()->cpu->invoke_scheduler = true; - thread_get_current_thread()->cpu->invoke_scheduler_if_idle = false; - thread_get_current_thread()->cpu->preempted = 1; + Thread* thread= thread_get_current_thread(); + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + + schedulerThreadData->lost_cpu = true; + thread->cpu->invoke_scheduler = true; + thread->cpu->invoke_scheduler_if_idle = false; + thread->cpu->preempted = 1; return B_HANDLED_INTERRUPT; } @@ -174,11 +260,18 @@ simple_reschedule(void) TRACE(("reschedule(): current thread = %ld\n", oldThread->id)); oldThread->state = oldThread->next_state; + scheduler_thread_data* schedulerOldThreadData + = reinterpret_cast(oldThread->scheduler_data); switch (oldThread->next_state) { case B_THREAD_RUNNING: case B_THREAD_READY: + if (oldThread->cpu->preempted) + simple_increase_penalty(oldThread); + else if (!schedulerOldThreadData->lost_cpu) + simple_cancel_penalty(oldThread); + TRACE(("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, oldThread->priority)); + oldThread->id, simple_get_effective_priority(oldThread))); simple_enqueue_in_run_queue(oldThread); break; case B_THREAD_SUSPENDED: @@ -192,46 +285,10 @@ simple_reschedule(void) break; } - while (true) { - // select thread with the biggest priority - nextThread = sRunQueue->PeekMaximum(); - if (!nextThread) - panic("reschedule(): run queue is empty!\n"); - -#if 0 - if (oldThread == nextThread && nextThread->was_yielded) { - // ignore threads that called thread_yield() once - nextThread->was_yielded = false; - prevThread = nextThread; - nextThread = nextThread->queue_next; - } -#endif - - // always extract real time threads - if (nextThread->priority >= B_FIRST_REAL_TIME_PRIORITY) - break; - - // find thread with the second biggest priority - Thread* lowerNextThread = sRunQueue->PeekSecondMaximum(); - - // never skip last non-idle normal thread - if (lowerNextThread == NULL - || lowerNextThread->priority == B_IDLE_PRIORITY) - break; - - int32 priorityDiff = nextThread->priority - lowerNextThread->priority; - if (priorityDiff > 15) - break; - - // skip normal threads sometimes - // (twice as probable per priority level) - if ((fast_random_value() >> (15 - priorityDiff)) != 0) - break; - - nextThread = lowerNextThread; - - break; - } + // select thread with the biggest priority + nextThread = sRunQueue->PeekMaximum(); + if (!nextThread) + panic("reschedule(): run queue is empty!\n"); sRunQueue->Remove(nextThread); @@ -244,6 +301,7 @@ simple_reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; oldThread->was_yielded = false; + schedulerOldThreadData->lost_cpu = false; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); @@ -282,7 +340,9 @@ simple_reschedule(void) static status_t simple_on_thread_create(Thread* thread, bool idleThread) { - // do nothing + thread->scheduler_data = new (std::nothrow)scheduler_thread_data; + if (thread->scheduler_data == NULL) + return B_NO_MEMORY; return B_OK; } @@ -290,14 +350,16 @@ simple_on_thread_create(Thread* thread, bool idleThread) static void simple_on_thread_init(Thread* thread) { - // do nothing + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + schedulerThreadData->Init(); } static void simple_on_thread_destroy(Thread* thread) { - // do nothing + delete thread->scheduler_data; } From 72f844835e8da9637448c49e4672e108b489d887 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 7 Oct 2013 21:29:10 +0200 Subject: [PATCH 017/273] kernel: Do not require double parentheses in TRACE statements. --- .../kernel/scheduler/scheduler_simple.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 6549dd42cc..e3bc662383 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -33,9 +33,9 @@ //#define TRACE_SCHEDULER #ifdef TRACE_SCHEDULER -# define TRACE(x) dprintf_no_syslog x +# define TRACE(x...) dprintf_no_syslog(x) #else -# define TRACE(x) ; +# define TRACE(x...) do { } while (false) #endif @@ -116,7 +116,7 @@ simple_increase_penalty(Thread *thread) if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) return; - TRACE(("increasing thread %ld penalty\n", thread->id)); + TRACE("increasing thread %ld penalty\n", thread->id); scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); @@ -134,7 +134,7 @@ simple_cancel_penalty(Thread *thread) scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); if (schedulerThreadData->priority_penalty != 0) - TRACE(("cancelling thread %ld penalty\n", thread->id)); + TRACE("cancelling thread %ld penalty\n", thread->id); schedulerThreadData->priority_penalty = 0; } @@ -257,7 +257,7 @@ simple_reschedule(void) } } - TRACE(("reschedule(): current thread = %ld\n", oldThread->id)); + TRACE("reschedule(): current thread = %ld\n", oldThread->id); oldThread->state = oldThread->next_state; scheduler_thread_data* schedulerOldThreadData @@ -270,18 +270,18 @@ simple_reschedule(void) else if (!schedulerOldThreadData->lost_cpu) simple_cancel_penalty(oldThread); - TRACE(("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, simple_get_effective_priority(oldThread))); + TRACE("enqueueing thread %ld into run queue priority = %ld\n", + oldThread->id, simple_get_effective_priority(oldThread)); simple_enqueue_in_run_queue(oldThread); break; case B_THREAD_SUSPENDED: - TRACE(("reschedule(): suspending thread %ld\n", oldThread->id)); + TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: break; default: - TRACE(("not enqueueing thread %ld into run queue next_state = %ld\n", - oldThread->id, oldThread->next_state)); + TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", + oldThread->id, oldThread->next_state); break; } From 565e7a977d988f78bae427456805aa6156b9b6ef Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 7 Oct 2013 21:52:45 +0200 Subject: [PATCH 018/273] kernel: Update SchedulerTracing::EnqueueThread --- src/system/kernel/scheduler/scheduler_affine.cpp | 2 +- src/system/kernel/scheduler/scheduler_simple.cpp | 4 ++-- .../kernel/scheduler/scheduler_simple_smp.cpp | 2 +- .../kernel/scheduler/scheduler_tracing.cpp | 4 ++-- src/system/kernel/scheduler/scheduler_tracing.h | 16 +++++----------- 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 078980ebf4..059df02654 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -162,7 +162,7 @@ affine_enqueue_in_run_queue(Thread *thread) prev = sRunQueue[targetCPU]; } - T(EnqueueThread(thread, prev, curr)); + T(EnqueueThread(thread, thread->priority)); sRunQueueSize[targetCPU]++; thread->queue_next = curr; if (prev) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index e3bc662383..da172b8237 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -147,9 +147,9 @@ simple_enqueue_in_run_queue(Thread *thread) { thread->state = thread->next_state = B_THREAD_READY; - //T(EnqueueThread(thread, prev, curr)); - int32 threadPriority = simple_get_effective_priority(thread); + T(EnqueueThread(thread, threadPriority)); + sRunQueue->PushBack(thread, threadPriority); thread->next_priority = thread->priority; diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp index 1a4a564353..66656120f0 100644 --- a/src/system/kernel/scheduler/scheduler_simple_smp.cpp +++ b/src/system/kernel/scheduler/scheduler_simple_smp.cpp @@ -126,7 +126,7 @@ enqueue_in_run_queue(Thread *thread) prev = sRunQueue; } - T(EnqueueThread(thread, prev, curr)); + T(EnqueueThread(thread, thread->priority)); thread->queue_next = curr; if (prev) diff --git a/src/system/kernel/scheduler/scheduler_tracing.cpp b/src/system/kernel/scheduler/scheduler_tracing.cpp index 858886fe8a..21eab8a0a8 100644 --- a/src/system/kernel/scheduler/scheduler_tracing.cpp +++ b/src/system/kernel/scheduler/scheduler_tracing.cpp @@ -19,8 +19,8 @@ namespace SchedulerTracing { void EnqueueThread::AddDump(TraceOutput& out) { - out.Print("scheduler enqueue %ld \"%s\", priority %d (previous %ld, " - "next %ld)", fID, fName, fPriority, fPreviousID, fNextID); + out.Print("scheduler enqueue %ld \"%s\", effective priority %d, " + "real priority %d", fID, fName, fEffectivePriority, fPriority); } diff --git a/src/system/kernel/scheduler/scheduler_tracing.h b/src/system/kernel/scheduler/scheduler_tracing.h index 422834d3d8..f537a6b6f3 100644 --- a/src/system/kernel/scheduler/scheduler_tracing.h +++ b/src/system/kernel/scheduler/scheduler_tracing.h @@ -36,17 +36,12 @@ protected: class EnqueueThread : public SchedulerTraceEntry { public: - EnqueueThread(Thread* thread, Thread* previous, Thread* next) + EnqueueThread(Thread* thread, int32 effectivePriority) : SchedulerTraceEntry(thread), - fPreviousID(-1), - fNextID(-1), - fPriority(thread->priority) + fPriority(thread->priority), + fEffectivePriority(effectivePriority) { - if (previous != NULL) - fPreviousID = previous->id; - if (next != NULL) - fNextID = next->id; fName = alloc_tracing_buffer_strcpy(thread->name, B_OS_NAME_LENGTH, false); Initialized(); @@ -57,10 +52,9 @@ public: virtual const char* Name() const; private: - thread_id fPreviousID; - thread_id fNextID; char* fName; - uint8 fPriority; + int32 fPriority; + int32 fEffectivePriority; }; From e083bca041629e35233a365cd93ed4da9fb8d0cd Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 01:08:05 +0200 Subject: [PATCH 019/273] kernel: Allow threads to always finish their time slice Until now, when the thread has been preempted by higher priority thread it was then placed at the end of its priority FIFO and given a new time slice. This patch changes it allowing the thread to complete its time slice (when the higher priority threads are done), unless there was very little time left in which case this time is added to the next time slice. Apart from making the algorithm more fair this change allows to identify CPU bound threads more easily. (Earlier they could 'hide' by being preempted by higher priority thread and consequently never using their whole time slice). --- .../kernel/scheduler/scheduler_simple.cpp | 101 +++++++++++++++--- 1 file changed, 85 insertions(+), 16 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index da172b8237..52e72f2d5d 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -47,11 +47,16 @@ static RunQueue* sRunQueue; struct scheduler_thread_data { - scheduler_thread_data() { Init(); } - void Init(); + scheduler_thread_data() { Init(); } + void Init(); - int32 priority_penalty; - bool lost_cpu; + int32 priority_penalty; + bool lost_cpu; + bool cpu_bound; + + bigtime_t time_left; + bigtime_t stolen_time; + bigtime_t quantum_start; }; @@ -59,7 +64,11 @@ void scheduler_thread_data::Init() { priority_penalty = 0; + time_left = 0; + stolen_time = 0; + lost_cpu = false; + cpu_bound = true; } @@ -150,8 +159,14 @@ simple_enqueue_in_run_queue(Thread *thread) int32 threadPriority = simple_get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + sRunQueue->PushBack(thread, threadPriority); thread->next_priority = thread->priority; + schedulerThreadData->cpu_bound = true; + schedulerThreadData->time_left = 0; + schedulerThreadData->stolen_time = 0; // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, @@ -159,11 +174,11 @@ simple_enqueue_in_run_queue(Thread *thread) Thread* currentThread = thread_get_current_thread(); if (threadPriority > currentThread->priority) { - scheduler_thread_data* schedulerThreadData + scheduler_thread_data* schedulerCurrentThreadData = reinterpret_cast( currentThread->scheduler_data); - schedulerThreadData->lost_cpu = true; + schedulerCurrentThreadData->lost_cpu = true; gCPU[0].invoke_scheduler = true; gCPU[0].invoke_scheduler_if_idle = false; } @@ -237,6 +252,46 @@ reschedule_event(timer *unused) } +static inline bool simple_quantum_ended(Thread* thread, bool wasPreempted) +{ + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + + bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; + schedulerThreadData->time_left -= time_used; + schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); + + // too little time left, it's better make the next quantum a bit longer + if (wasPreempted || schedulerThreadData->time_left <= kThreadQuantum / 50) { + schedulerThreadData->stolen_time += schedulerThreadData->time_left; + schedulerThreadData->time_left = 0; + } + + return schedulerThreadData->time_left == 0; +} + + +static inline bigtime_t simple_compute_quantum(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + + bigtime_t quantum; + if (schedulerThreadData->time_left != 0) + quantum = schedulerThreadData->time_left; + else + quantum = kThreadQuantum; + + quantum += schedulerThreadData->stolen_time; + schedulerThreadData->stolen_time = 0; + + schedulerThreadData->time_left = quantum; + schedulerThreadData->quantum_start = system_time(); + + return quantum; +} + + /*! Runs the scheduler. Note: expects thread spinlock to be held */ @@ -262,29 +317,45 @@ simple_reschedule(void) oldThread->state = oldThread->next_state; scheduler_thread_data* schedulerOldThreadData = reinterpret_cast(oldThread->scheduler_data); + switch (oldThread->next_state) { case B_THREAD_RUNNING: case B_THREAD_READY: - if (oldThread->cpu->preempted) - simple_increase_penalty(oldThread); - else if (!schedulerOldThreadData->lost_cpu) - simple_cancel_penalty(oldThread); + if (!schedulerOldThreadData->lost_cpu) + schedulerOldThreadData->cpu_bound = false; + + if (simple_quantum_ended(oldThread, oldThread->cpu->preempted)) { + if (schedulerOldThreadData->cpu_bound) + simple_increase_penalty(oldThread); + else + simple_cancel_penalty(oldThread); + + TRACE("enqueueing thread %ld into run queue priority = %ld\n", + oldThread->id, simple_get_effective_priority(oldThread)); + simple_enqueue_in_run_queue(oldThread); + } else { + TRACE("putting thread %ld back in run queue priority = %ld\n", + oldThread->id, simple_get_effective_priority(oldThread)); + sRunQueue->PushFront(oldThread, + simple_get_effective_priority(oldThread)); + } - TRACE("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, simple_get_effective_priority(oldThread)); - simple_enqueue_in_run_queue(oldThread); break; case B_THREAD_SUSPENDED: + simple_cancel_penalty(oldThread); TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: break; default: + simple_cancel_penalty(oldThread); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, oldThread->next_state); break; } + schedulerOldThreadData->lost_cpu = false; + // select thread with the biggest priority nextThread = sRunQueue->PeekMaximum(); if (!nextThread) @@ -301,7 +372,6 @@ simple_reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; oldThread->was_yielded = false; - schedulerOldThreadData->lost_cpu = false; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); @@ -319,14 +389,13 @@ simple_reschedule(void) } if (nextThread != oldThread || oldThread->cpu->preempted) { - bigtime_t quantum = kThreadQuantum; // TODO: calculate quantum? timer* quantumTimer = &oldThread->cpu->quantum_timer; - if (!oldThread->cpu->preempted) cancel_timer(quantumTimer); oldThread->cpu->preempted = 0; if (!thread_is_idle_thread(nextThread)) { + bigtime_t quantum = simple_compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); } From 31e65090db03c13d04adcfcc4cb85aba04cb536f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 01:31:05 +0200 Subject: [PATCH 020/273] kernel: Use standard compliant version of variadic macros --- src/system/kernel/scheduler/scheduler_simple.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 52e72f2d5d..417e90843a 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -33,9 +33,9 @@ //#define TRACE_SCHEDULER #ifdef TRACE_SCHEDULER -# define TRACE(x...) dprintf_no_syslog(x) +# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) #else -# define TRACE(x...) do { } while (false) +# define TRACE(...) do { } while (false) #endif From 4cba4ff1df8b1dd778bf092f9c1936ff8385ed88 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 01:32:53 +0200 Subject: [PATCH 021/273] kernel/util: Use private/kernel/debug.h --- headers/private/kernel/util/Bitmap.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/headers/private/kernel/util/Bitmap.h b/headers/private/kernel/util/Bitmap.h index b47854cc61..08114715db 100644 --- a/headers/private/kernel/util/Bitmap.h +++ b/headers/private/kernel/util/Bitmap.h @@ -9,7 +9,8 @@ #define KERNEL_UTIL_BITMAP_H -#include +#include + #include From 21808e8f0bf71d342e572e9980119c0a5dc24ec6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 02:54:58 +0200 Subject: [PATCH 022/273] kernel: Limit maximum priority penalty The maximum penalty the thread can receive is now limited depending on the real thread priority. However, since it make it possible to starve threads with priority lower than that limit. To prevent that threads that have already earned the maximum penalty are periodically forced to yield CPU to all other threads. --- .../kernel/scheduler/scheduler_simple.cpp | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 417e90843a..461c601f52 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -51,6 +51,8 @@ struct scheduler_thread_data { void Init(); int32 priority_penalty; + int32 forced_yield_count; + bool lost_cpu; bool cpu_bound; @@ -64,6 +66,8 @@ void scheduler_thread_data::Init() { priority_penalty = 0; + forced_yield_count = 0; + time_left = 0; stolen_time = 0; @@ -107,8 +111,14 @@ simple_get_effective_priority(Thread *thread) = reinterpret_cast(thread->scheduler_data); int32 effectivePriority = thread->priority; - if (effectivePriority < B_FIRST_REAL_TIME_PRIORITY) - effectivePriority -= schedulerThreadData->priority_penalty; + if (effectivePriority < B_FIRST_REAL_TIME_PRIORITY) { + if (schedulerThreadData->forced_yield_count + && schedulerThreadData->forced_yield_count % 16 == 0) { + TRACE("forcing thread %ld to yield\n", thread->id); + effectivePriority = B_LOWEST_ACTIVE_PRIORITY; + } else + effectivePriority -= schedulerThreadData->priority_penalty; + } ASSERT(schedulerThreadData->priority_penalty >= 0); ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); @@ -132,8 +142,12 @@ simple_increase_penalty(Thread *thread) int32 oldPenalty = schedulerThreadData->priority_penalty++; ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - if (thread->priority - oldPenalty <= B_LOWEST_ACTIVE_PRIORITY) + const int kMinimalPriority + = min_c(thread->priority, 25) / 5; + if (thread->priority - oldPenalty <= kMinimalPriority) { schedulerThreadData->priority_penalty = oldPenalty; + schedulerThreadData->forced_yield_count++; + } } From 547b8c76c718b46174dbac0ae37d3ad6b9d803b5 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 04:50:23 +0200 Subject: [PATCH 023/273] kernel: Cancel penalty only if the thread actually waits Require the thread to give up CPU for at least one time slice before cancelling its penalty. --- .../kernel/scheduler/scheduler_simple.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 461c601f52..c8517b64e1 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -59,6 +59,8 @@ struct scheduler_thread_data { bigtime_t time_left; bigtime_t stolen_time; bigtime_t quantum_start; + + bigtime_t went_sleep; }; @@ -71,6 +73,8 @@ scheduler_thread_data::Init() time_left = 0; stolen_time = 0; + went_sleep = 0; + lost_cpu = false; cpu_bound = true; } @@ -156,9 +160,14 @@ simple_cancel_penalty(Thread *thread) { scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); + + if (schedulerThreadData->went_sleep < 0 + || system_time() - schedulerThreadData->went_sleep <= kThreadQuantum) + return; if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); schedulerThreadData->priority_penalty = 0; + schedulerThreadData->forced_yield_count = 0; } @@ -170,6 +179,8 @@ simple_enqueue_in_run_queue(Thread *thread) { thread->state = thread->next_state = B_THREAD_READY; + simple_cancel_penalty(thread); + int32 threadPriority = simple_get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); @@ -339,6 +350,8 @@ simple_reschedule(void) schedulerOldThreadData->cpu_bound = false; if (simple_quantum_ended(oldThread, oldThread->cpu->preempted)) { + schedulerOldThreadData->went_sleep = -1; + if (schedulerOldThreadData->cpu_bound) simple_increase_penalty(oldThread); else @@ -356,13 +369,13 @@ simple_reschedule(void) break; case B_THREAD_SUSPENDED: - simple_cancel_penalty(oldThread); + schedulerOldThreadData->went_sleep = system_time(); TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: break; default: - simple_cancel_penalty(oldThread); + schedulerOldThreadData->went_sleep = system_time(); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, oldThread->next_state); break; From bab69bdb4707498efc97bb3bf3d58d58553b4dbc Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 04:53:30 +0200 Subject: [PATCH 024/273] kernel: Force high priority threads to yield less often --- src/system/kernel/scheduler/scheduler_simple.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index c8517b64e1..950da57013 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -116,8 +116,9 @@ simple_get_effective_priority(Thread *thread) int32 effectivePriority = thread->priority; if (effectivePriority < B_FIRST_REAL_TIME_PRIORITY) { - if (schedulerThreadData->forced_yield_count - && schedulerThreadData->forced_yield_count % 16 == 0) { + const int kYieldFrequency = 1 << (min_c(thread->priority, 25) / 5 + 1); + if (schedulerThreadData->forced_yield_count != 0 + && schedulerThreadData->forced_yield_count % kYieldFrequency == 0) { TRACE("forcing thread %ld to yield\n", thread->id); effectivePriority = B_LOWEST_ACTIVE_PRIORITY; } else From 03e3a8295375a672734943bcaa489ad09b22ac35 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 06:41:20 +0200 Subject: [PATCH 025/273] kernel: Allow threads to yield CPU properly --- .../kernel/scheduler/scheduler_simple.cpp | 92 +++++++++++++++---- src/system/kernel/thread.cpp | 9 +- 2 files changed, 75 insertions(+), 26 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 950da57013..db51e536ee 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -43,7 +44,10 @@ const bigtime_t kThreadQuantum = 3000; // The run queue. Holds the threads ready to run ordered by priority. -static RunQueue* sRunQueue; +typedef RunQueue SimpleRunQueue; +static SimpleRunQueue* sRunQueue; +static SimpleRunQueue* sExpiredQueue; +static int32 sYieldedThreadPriority = -1; struct scheduler_thread_data { @@ -80,16 +84,13 @@ scheduler_thread_data::Init() } -static int -dump_run_queue(int argc, char **argv) +static inline void +dump_queue(RunQueue::ConstIterator& iterator) { - RunQueue::ConstIterator iterator; - iterator = sRunQueue->GetConstIterator(); - if (!iterator.HasNext()) - kprintf("Run queue is empty!\n"); + kprintf("Queue is empty.\n"); else { - kprintf("thread id priority penalty name\n"); + kprintf("thread id priority penalty name\n"); while (iterator.HasNext()) { Thread *thread = iterator.Next(); scheduler_thread_data* schedulerThreadData @@ -100,11 +101,33 @@ dump_run_queue(int argc, char **argv) schedulerThreadData->priority_penalty, thread->name); } } +} + + +static int +dump_run_queue(int argc, char **argv) +{ + RunQueue::ConstIterator iterator; + kprintf("Current run queue:\n"); + iterator = sRunQueue->GetConstIterator(); + dump_queue(iterator); + + kprintf("\nExpired run queue:\n"); + iterator = sExpiredQueue->GetConstIterator(); + dump_queue(iterator); return 0; } +static inline void +simple_yield(Thread* thread) +{ + TRACE("thread %ld yielded\n", thread->id); + sYieldedThreadPriority = max_c(sYieldedThreadPriority, thread->priority); +} + + static inline int32 simple_get_effective_priority(Thread *thread) { @@ -120,9 +143,10 @@ simple_get_effective_priority(Thread *thread) if (schedulerThreadData->forced_yield_count != 0 && schedulerThreadData->forced_yield_count % kYieldFrequency == 0) { TRACE("forcing thread %ld to yield\n", thread->id); - effectivePriority = B_LOWEST_ACTIVE_PRIORITY; - } else - effectivePriority -= schedulerThreadData->priority_penalty; + simple_yield(thread); + } + + effectivePriority -= schedulerThreadData->priority_penalty; } ASSERT(schedulerThreadData->priority_penalty >= 0); @@ -188,7 +212,11 @@ simple_enqueue_in_run_queue(Thread *thread) scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); - sRunQueue->PushBack(thread, threadPriority); + if (threadPriority <= sYieldedThreadPriority) + sExpiredQueue->PushBack(thread, threadPriority); + else + sRunQueue->PushBack(thread, threadPriority); + thread->next_priority = thread->priority; schedulerThreadData->cpu_bound = true; schedulerThreadData->time_left = 0; @@ -358,6 +386,10 @@ simple_reschedule(void) else simple_cancel_penalty(oldThread); + if (oldThread->was_yielded) + simple_yield(oldThread); + oldThread->was_yielded = false; + TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, simple_get_effective_priority(oldThread)); simple_enqueue_in_run_queue(oldThread); @@ -385,7 +417,26 @@ simple_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - nextThread = sRunQueue->PeekMaximum(); + do { + nextThread = sRunQueue->PeekMaximum(); + + if (sYieldedThreadPriority >= 0 && nextThread != NULL + && thread_is_idle_thread(nextThread)) { + sRunQueue->Remove(nextThread); + simple_enqueue_in_run_queue(nextThread); + continue; + } + + break; + } while (true); + if (nextThread == NULL && sYieldedThreadPriority >= 0) { + RunQueue* temp = sRunQueue; + sRunQueue = sExpiredQueue; + sExpiredQueue = temp; + sYieldedThreadPriority = -1; + + nextThread = sRunQueue->PeekMaximum(); + } if (!nextThread) panic("reschedule(): run queue is empty!\n"); @@ -399,7 +450,6 @@ simple_reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; - oldThread->was_yielded = false; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); @@ -490,20 +540,26 @@ static scheduler_ops kSimpleOps = { status_t scheduler_simple_init() { - sRunQueue = new(std::nothrow) RunQueue; + sRunQueue = new(std::nothrow) SimpleRunQueue; if (sRunQueue == NULL) return B_NO_MEMORY; + ObjectDeleter runQueueDeleter(sRunQueue); + + sExpiredQueue = new(std::nothrow) SimpleRunQueue; + if (sExpiredQueue == NULL) + return B_NO_MEMORY; + ObjectDeleter expiredQueueDeleter(sExpiredQueue); status_t result = sRunQueue->GetInitStatus(); - if (result != B_OK) { - delete sRunQueue; + if (result != B_OK) return result; - } gScheduler = &kSimpleOps; add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); + runQueueDeleter.Detach(); + expiredQueueDeleter.Detach(); return B_OK; } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 7e29df2927..0c785a22a5 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2447,11 +2447,6 @@ void thread_yield(bool force) { if (force) { - // snooze for roughly 3 thread quantums - snooze_etc(9000, B_SYSTEM_TIMEBASE, B_RELATIVE_TIMEOUT | B_CAN_INTERRUPT); -#if 0 - cpu_status state; - Thread *thread = thread_get_current_thread(); if (thread == NULL) return; @@ -2459,10 +2454,8 @@ thread_yield(bool force) InterruptsSpinLocker _(gSchedulerLock); // mark the thread as yielded, so it will not be scheduled next - //thread->was_yielded = true; - thread->next_priority = B_LOWEST_ACTIVE_PRIORITY; + thread->was_yielded = true; scheduler_reschedule(); -#endif } else { Thread *thread = thread_get_current_thread(); if (thread == NULL) From 94f4574d78aca6445d60c0c04105f91b1ff01558 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 20:13:10 +0200 Subject: [PATCH 026/273] kernel: Move thread retrieving code to separate function --- .../kernel/scheduler/scheduler_simple.cpp | 54 +++++++++++-------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index db51e536ee..2cafe65dd2 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -346,6 +346,35 @@ static inline bigtime_t simple_compute_quantum(Thread* thread) } +static inline Thread* +simple_get_next_thread(void) +{ + Thread* thread; + do { + thread = sRunQueue->PeekMaximum(); + + if (sYieldedThreadPriority >= 0 && thread != NULL + && thread_is_idle_thread(thread)) { + sRunQueue->Remove(thread); + simple_enqueue_in_run_queue(thread); + continue; + } + + break; + } while (true); + if (thread == NULL && sYieldedThreadPriority >= 0) { + SimpleRunQueue* temp = sRunQueue; + sRunQueue = sExpiredQueue; + sExpiredQueue = temp; + sYieldedThreadPriority = -1; + + thread = sRunQueue->PeekMaximum(); + } + + return thread; +} + + /*! Runs the scheduler. Note: expects thread spinlock to be held */ @@ -353,7 +382,6 @@ static void simple_reschedule(void) { Thread* oldThread = thread_get_current_thread(); - Thread* nextThread; // check whether we're only supposed to reschedule, if the current thread // is idle @@ -417,29 +445,9 @@ simple_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - do { - nextThread = sRunQueue->PeekMaximum(); - - if (sYieldedThreadPriority >= 0 && nextThread != NULL - && thread_is_idle_thread(nextThread)) { - sRunQueue->Remove(nextThread); - simple_enqueue_in_run_queue(nextThread); - continue; - } - - break; - } while (true); - if (nextThread == NULL && sYieldedThreadPriority >= 0) { - RunQueue* temp = sRunQueue; - sRunQueue = sExpiredQueue; - sExpiredQueue = temp; - sYieldedThreadPriority = -1; - - nextThread = sRunQueue->PeekMaximum(); - } + Thread* nextThread = simple_get_next_thread(); if (!nextThread) - panic("reschedule(): run queue is empty!\n"); - + panic("reschedule(): run queues are empty!\n"); sRunQueue->Remove(nextThread); T(ScheduleThread(nextThread, oldThread)); From 346e789a21882595ca008f58303ab76787691dc7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 20:15:21 +0200 Subject: [PATCH 027/273] kernel: Fix style issues --- src/system/kernel/scheduler/scheduler_simple.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 2cafe65dd2..3601c0a6a4 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -92,7 +92,7 @@ dump_queue(RunQueue::ConstIterator& iterator) else { kprintf("thread id priority penalty name\n"); while (iterator.HasNext()) { - Thread *thread = iterator.Next(); + Thread* thread = iterator.Next(); scheduler_thread_data* schedulerThreadData = reinterpret_cast( thread->scheduler_data); @@ -105,7 +105,7 @@ dump_queue(RunQueue::ConstIterator& iterator) static int -dump_run_queue(int argc, char **argv) +dump_run_queue(int argc, char** argv) { RunQueue::ConstIterator iterator; kprintf("Current run queue:\n"); @@ -129,7 +129,7 @@ simple_yield(Thread* thread) static inline int32 -simple_get_effective_priority(Thread *thread) +simple_get_effective_priority(Thread* thread) { if (thread->priority == B_IDLE_PRIORITY) return thread->priority; @@ -157,7 +157,7 @@ simple_get_effective_priority(Thread *thread) static inline void -simple_increase_penalty(Thread *thread) +simple_increase_penalty(Thread* thread) { if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) return; @@ -181,7 +181,7 @@ simple_increase_penalty(Thread *thread) static inline void -simple_cancel_penalty(Thread *thread) +simple_cancel_penalty(Thread* thread) { scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); @@ -200,7 +200,7 @@ simple_cancel_penalty(Thread *thread) Note: thread lock must be held when entering this function */ static void -simple_enqueue_in_run_queue(Thread *thread) +simple_enqueue_in_run_queue(Thread* thread) { thread->state = thread->next_state = B_THREAD_READY; @@ -243,7 +243,7 @@ simple_enqueue_in_run_queue(Thread *thread) Note: thread lock must be held when entering this function */ static void -simple_set_thread_priority(Thread *thread, int32 priority) +simple_set_thread_priority(Thread* thread, int32 priority) { if (priority == thread->priority) return; @@ -290,7 +290,7 @@ simple_estimate_max_scheduling_latency(Thread* thread) static int32 -reschedule_event(timer *unused) +reschedule_event(timer* /* unused */) { // This function is called as a result of the timer event set by the // scheduler. Make sure the reschedule() is invoked. From 9363e99b19d122db7d6684b06d7240fb73255cd2 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 20:21:35 +0200 Subject: [PATCH 028/273] kernel: Remove Thread::next_priority --- headers/private/kernel/thread_types.h | 1 - .../kernel/scheduler/scheduler_affine.cpp | 6 ++---- .../kernel/scheduler/scheduler_simple.cpp | 5 ++--- .../kernel/scheduler/scheduler_simple_smp.cpp | 6 ++---- src/system/kernel/thread.cpp | 17 +++++++---------- 5 files changed, 13 insertions(+), 22 deletions(-) diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 637c5bed42..3e4ec90261 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -422,7 +422,6 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, timer alarm; // protected by scheduler lock char name[B_OS_NAME_LENGTH]; // protected by fLock int32 priority; // protected by scheduler lock - int32 next_priority; // protected by scheduler lock int32 io_priority; // protected by fLock int32 state; // protected by scheduler lock int32 next_state; // protected by scheduler lock diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 059df02654..f5b7294b10 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -154,7 +154,7 @@ affine_enqueue_in_run_queue(Thread *thread) } else { Thread *curr, *prev; for (curr = sRunQueue[targetCPU], prev = NULL; curr - && curr->priority >= thread->next_priority; + && curr->priority >= thread->priority; curr = curr->queue_next) { if (prev) prev = prev->queue_next; @@ -173,8 +173,6 @@ affine_enqueue_in_run_queue(Thread *thread) thread->scheduler_data->fLastQueue = targetCPU; } - thread->next_priority = thread->priority; - // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); @@ -304,7 +302,7 @@ affine_set_thread_priority(Thread *thread, int32 priority) thread = dequeue_from_run_queue(prev, targetCPU); // set priority and re-insert - thread->priority = thread->next_priority = priority; + thread->priority = priority; affine_enqueue_in_run_queue(thread); } diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 3601c0a6a4..99818d8f4e 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -152,7 +152,7 @@ simple_get_effective_priority(Thread* thread) ASSERT(schedulerThreadData->priority_penalty >= 0); ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - return min_c(effectivePriority, thread->next_priority); + return effectivePriority; } @@ -217,7 +217,6 @@ simple_enqueue_in_run_queue(Thread* thread) else sRunQueue->PushBack(thread, threadPriority); - thread->next_priority = thread->priority; schedulerThreadData->cpu_bound = true; schedulerThreadData->time_left = 0; schedulerThreadData->stolen_time = 0; @@ -267,7 +266,7 @@ simple_set_thread_priority(Thread* thread, int32 priority) // set priority and re-insert simple_cancel_penalty(thread); - thread->priority = thread->next_priority = priority; + thread->priority = priority; simple_enqueue_in_run_queue(thread); } diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp index 66656120f0..092fd70176 100644 --- a/src/system/kernel/scheduler/scheduler_simple_smp.cpp +++ b/src/system/kernel/scheduler/scheduler_simple_smp.cpp @@ -118,7 +118,7 @@ enqueue_in_run_queue(Thread *thread) Thread *curr, *prev; for (curr = sRunQueue, prev = NULL; curr - && curr->priority >= thread->next_priority; + && curr->priority >= thread->priority; curr = curr->queue_next) { if (prev) prev = prev->queue_next; @@ -134,8 +134,6 @@ enqueue_in_run_queue(Thread *thread) else sRunQueue = thread; - thread->next_priority = thread->priority; - if (thread->priority != B_IDLE_PRIORITY) { // Select a CPU for the thread to run on. It's not certain that the // thread will actually run on it, but we will notify the CPU to @@ -211,7 +209,7 @@ set_thread_priority(Thread *thread, int32 priority) sRunQueue = item->queue_next; // set priority and re-insert - thread->priority = thread->next_priority = priority; + thread->priority = priority; enqueue_in_run_queue(thread); } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 0c785a22a5..e2dc89c439 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -167,7 +167,6 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) team_next(NULL), queue_next(NULL), priority(-1), - next_priority(-1), io_priority(-1), cpu(cpu), previous_cpu(NULL), @@ -901,7 +900,6 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel) // available for deinitialization thread->priority = attributes.priority == -1 ? B_NORMAL_PRIORITY : attributes.priority; - thread->next_priority = thread->priority; thread->state = B_THREAD_SUSPENDED; thread->next_state = B_THREAD_SUSPENDED; @@ -1427,7 +1425,7 @@ make_thread_unreal(int argc, char **argv) continue; if (thread->priority > B_DISPLAY_PRIORITY) { - thread->priority = thread->next_priority = B_NORMAL_PRIORITY; + thread->priority = B_NORMAL_PRIORITY; kprintf("thread %" B_PRId32 " made unreal\n", thread->id); } } @@ -1463,7 +1461,7 @@ set_thread_prio(int argc, char **argv) Thread* thread = it.Next();) { if (thread->id != id) continue; - thread->priority = thread->next_priority = prio; + thread->priority = prio; kprintf("thread %" B_PRId32 " set to priority %" B_PRId32 "\n", id, prio); found = true; break; @@ -1702,9 +1700,8 @@ _dump_thread_info(Thread *thread, bool shortInfo) kprintf("name: \"%s\"\n", thread->name); kprintf("hash_next: %p\nteam_next: %p\nq_next: %p\n", thread->hash_next, thread->team_next, thread->queue_next); - kprintf("priority: %" B_PRId32 " (next %" B_PRId32 ", " - "I/O: %" B_PRId32 ")\n", thread->priority, thread->next_priority, - thread->io_priority); + kprintf("priority: %" B_PRId32 " (I/O: %" B_PRId32 ")\n", + thread->priority, thread->io_priority); kprintf("state: %s\n", state_to_text(thread, thread->state)); kprintf("next_state: %s\n", state_to_text(thread, thread->next_state)); kprintf("cpu: %p ", thread->cpu); @@ -1919,7 +1916,7 @@ thread_exit(void) panic("thread_exit() called with interrupts disabled!\n"); // boost our priority to get this over with - thread->priority = thread->next_priority = B_URGENT_DISPLAY_PRIORITY; + thread->priority = B_URGENT_DISPLAY_PRIORITY; if (team != kernelTeam) { // Cancel previously installed alarm timer, if any. Hold the scheduler @@ -2730,7 +2727,7 @@ thread_init(kernel_args *args) gCPU[i].running_thread = thread; thread->team = team_get_kernel_team(); - thread->priority = thread->next_priority = B_IDLE_PRIORITY; + thread->priority = B_IDLE_PRIORITY; thread->state = B_THREAD_RUNNING; thread->next_state = B_THREAD_READY; sprintf(name, "idle thread %" B_PRIu32 " kstack", i + 1); @@ -3217,7 +3214,7 @@ set_thread_priority(thread_id id, int32 priority) // It's ourself, so we know we aren't in the run queue, and we can // manipulate our structure directly. oldPriority = thread->priority; - thread->priority = thread->next_priority = priority; + thread->priority = priority; } else { oldPriority = thread->priority; scheduler_set_thread_priority(thread, priority); From ee69e536309045ba557e9defc2436eefb06d88d3 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 8 Oct 2013 21:36:49 +0200 Subject: [PATCH 029/273] kernel: Minor improvements, separate priority and yield logic --- .../kernel/scheduler/scheduler_simple.cpp | 89 +++++++++++-------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 99818d8f4e..bf1a402e0b 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -120,42 +120,52 @@ dump_run_queue(int argc, char** argv) } -static inline void -simple_yield(Thread* thread) -{ - TRACE("thread %ld yielded\n", thread->id); - sYieldedThreadPriority = max_c(sYieldedThreadPriority, thread->priority); -} - - static inline int32 simple_get_effective_priority(Thread* thread) { if (thread->priority == B_IDLE_PRIORITY) return thread->priority; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return thread->priority; scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); int32 effectivePriority = thread->priority; - if (effectivePriority < B_FIRST_REAL_TIME_PRIORITY) { - const int kYieldFrequency = 1 << (min_c(thread->priority, 25) / 5 + 1); - if (schedulerThreadData->forced_yield_count != 0 - && schedulerThreadData->forced_yield_count % kYieldFrequency == 0) { - TRACE("forcing thread %ld to yield\n", thread->id); - simple_yield(thread); - } + effectivePriority -= schedulerThreadData->priority_penalty; - effectivePriority -= schedulerThreadData->priority_penalty; - } - - ASSERT(schedulerThreadData->priority_penalty >= 0); + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); return effectivePriority; } +static inline void +simple_yield(Thread* thread) +{ + TRACE("thread %ld yielded\n", thread->id); + int32 effectivePriority = simple_get_effective_priority(thread); + sYieldedThreadPriority = max_c(sYieldedThreadPriority, effectivePriority); +} + + +static inline bool +simple_should_force_yield(Thread* thread) +{ + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return false; + + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + + const int kYieldFrequency = 1 << (min_c(thread->priority, 25) / 5 + 1); + + return schedulerThreadData->forced_yield_count != 0 + && schedulerThreadData->forced_yield_count % kYieldFrequency == 0; +} + + static inline void simple_increase_penalty(Thread* thread) { @@ -186,9 +196,6 @@ simple_cancel_penalty(Thread* thread) scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); - if (schedulerThreadData->went_sleep < 0 - || system_time() - schedulerThreadData->went_sleep <= kThreadQuantum) - return; if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); schedulerThreadData->priority_penalty = 0; @@ -196,22 +203,24 @@ simple_cancel_penalty(Thread* thread) } -/*! Enqueues the thread into the run queue. - Note: thread lock must be held when entering this function -*/ static void -simple_enqueue_in_run_queue(Thread* thread) +simple_enqueue(Thread* thread, bool newOne) { thread->state = thread->next_state = B_THREAD_READY; - simple_cancel_penalty(thread); + scheduler_thread_data* schedulerThreadData + = reinterpret_cast(thread->scheduler_data); + + bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; + if (newOne && hasSlept > kThreadQuantum) + simple_cancel_penalty(thread); + + if (simple_should_force_yield(thread)) + simple_yield(thread); int32 threadPriority = simple_get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); - if (threadPriority <= sYieldedThreadPriority) sExpiredQueue->PushBack(thread, threadPriority); else @@ -226,7 +235,7 @@ simple_enqueue_in_run_queue(Thread* thread) thread); Thread* currentThread = thread_get_current_thread(); - if (threadPriority > currentThread->priority) { + if (newOne && threadPriority > currentThread->priority) { scheduler_thread_data* schedulerCurrentThreadData = reinterpret_cast( currentThread->scheduler_data); @@ -238,6 +247,16 @@ simple_enqueue_in_run_queue(Thread* thread) } +/*! Enqueues the thread into the run queue. + Note: thread lock must be held when entering this function +*/ +static void +simple_enqueue_in_run_queue(Thread* thread) +{ + simple_enqueue(thread, true); +} + + /*! Sets the priority of a thread. Note: thread lock must be held when entering this function */ @@ -406,8 +425,6 @@ simple_reschedule(void) schedulerOldThreadData->cpu_bound = false; if (simple_quantum_ended(oldThread, oldThread->cpu->preempted)) { - schedulerOldThreadData->went_sleep = -1; - if (schedulerOldThreadData->cpu_bound) simple_increase_penalty(oldThread); else @@ -415,11 +432,10 @@ simple_reschedule(void) if (oldThread->was_yielded) simple_yield(oldThread); - oldThread->was_yielded = false; TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, simple_get_effective_priority(oldThread)); - simple_enqueue_in_run_queue(oldThread); + simple_enqueue(oldThread, false); } else { TRACE("putting thread %ld back in run queue priority = %ld\n", oldThread->id, simple_get_effective_priority(oldThread)); @@ -441,6 +457,7 @@ simple_reschedule(void) break; } + oldThread->was_yielded = false; schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority @@ -449,6 +466,8 @@ simple_reschedule(void) panic("reschedule(): run queues are empty!\n"); sRunQueue->Remove(nextThread); + TRACE("reschedule(): next thread = %ld\n", nextThread->id); + T(ScheduleThread(nextThread, oldThread)); // notify listeners From 0896565a6e405d3543b15772571138d622c7fba9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 01:18:55 +0200 Subject: [PATCH 030/273] kernel: Support sched_yield() properly sched_yield() should not yield to the threads with lower priority. --- headers/private/kernel/thread_types.h | 3 +- .../kernel/scheduler/scheduler_affine.cpp | 2 +- .../kernel/scheduler/scheduler_simple.cpp | 21 +++++++++----- .../kernel/scheduler/scheduler_simple_smp.cpp | 2 +- src/system/kernel/thread.cpp | 29 +++++++------------ 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 3e4ec90261..23e10e9138 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -443,7 +443,8 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, bool in_kernel; // protected by time_lock, only written by // this thread - bool was_yielded; // protected by scheduler lock + bool has_yielded; // protected by scheduler lock + bool has_fully_yielded; // protected by scheduler lock struct scheduler_thread_data* scheduler_data; // protected by scheduler lock struct user_thread* user_thread; // write-protected by fLock, only diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index f5b7294b10..fde10f0d59 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -447,7 +447,7 @@ affine_reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; - oldThread->was_yielded = false; + oldThread->has_yielded = false; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index bf1a402e0b..de317f1cb6 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -324,11 +324,17 @@ reschedule_event(timer* /* unused */) } -static inline bool simple_quantum_ended(Thread* thread, bool wasPreempted) +static inline bool +simple_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) { scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); + if (hasYielded) { + schedulerThreadData->time_left = 0; + return true; + } + bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; schedulerThreadData->time_left -= time_used; schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); @@ -343,7 +349,8 @@ static inline bool simple_quantum_ended(Thread* thread, bool wasPreempted) } -static inline bigtime_t simple_compute_quantum(Thread* thread) +static inline bigtime_t +simple_compute_quantum(Thread* thread) { scheduler_thread_data* schedulerThreadData = reinterpret_cast(thread->scheduler_data); @@ -424,13 +431,12 @@ simple_reschedule(void) if (!schedulerOldThreadData->lost_cpu) schedulerOldThreadData->cpu_bound = false; - if (simple_quantum_ended(oldThread, oldThread->cpu->preempted)) { + if (simple_quantum_ended(oldThread, oldThread->cpu->preempted, + oldThread->has_yielded)) { if (schedulerOldThreadData->cpu_bound) simple_increase_penalty(oldThread); - else - simple_cancel_penalty(oldThread); - if (oldThread->was_yielded) + if (oldThread->has_fully_yielded) simple_yield(oldThread); TRACE("enqueueing thread %ld into run queue priority = %ld\n", @@ -457,7 +463,8 @@ simple_reschedule(void) break; } - oldThread->was_yielded = false; + oldThread->has_yielded = false; + oldThread->has_fully_yielded = false; schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp index 092fd70176..d30ce9c3a7 100644 --- a/src/system/kernel/scheduler/scheduler_simple_smp.cpp +++ b/src/system/kernel/scheduler/scheduler_simple_smp.cpp @@ -384,7 +384,7 @@ reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; - oldThread->was_yielded = false; + oldThread->has_yielded = false; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index e2dc89c439..781720c1c6 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -178,7 +178,8 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) signal_stack_size(0), signal_stack_enabled(false), in_kernel(true), - was_yielded(false), + has_yielded(false), + has_fully_yielded(false), user_thread(NULL), fault_handler(0), page_faults_allowed(1), @@ -2443,25 +2444,15 @@ peek_next_thread_id() void thread_yield(bool force) { - if (force) { - Thread *thread = thread_get_current_thread(); - if (thread == NULL) - return; + Thread *thread = thread_get_current_thread(); + if (thread == NULL) + return; - InterruptsSpinLocker _(gSchedulerLock); + InterruptsSpinLocker _(gSchedulerLock); - // mark the thread as yielded, so it will not be scheduled next - thread->was_yielded = true; - scheduler_reschedule(); - } else { - Thread *thread = thread_get_current_thread(); - if (thread == NULL) - return; - - // Don't force the thread off the CPU, just reschedule. - InterruptsSpinLocker _(gSchedulerLock); - scheduler_reschedule(); - } + thread->has_yielded = true; + thread->has_fully_yielded = force; + scheduler_reschedule(); } @@ -3512,7 +3503,7 @@ _user_snooze_etc(bigtime_t timeout, int timebase, uint32 flags, void _user_thread_yield(void) { - thread_yield(true); + thread_yield(false); } From f256b4aca7aba52cafb5f405557d3470218f61a1 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 01:20:40 +0200 Subject: [PATCH 031/273] kernel: Use SimpleRunQueue as run queue type everywhere --- src/system/kernel/scheduler/scheduler_simple.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index de317f1cb6..40a789aa98 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -85,7 +85,7 @@ scheduler_thread_data::Init() static inline void -dump_queue(RunQueue::ConstIterator& iterator) +dump_queue(SimpleRunQueue::ConstIterator& iterator) { if (!iterator.HasNext()) kprintf("Queue is empty.\n"); @@ -107,7 +107,7 @@ dump_queue(RunQueue::ConstIterator& iterator) static int dump_run_queue(int argc, char** argv) { - RunQueue::ConstIterator iterator; + SimpleRunQueue::ConstIterator iterator; kprintf("Current run queue:\n"); iterator = sRunQueue->GetConstIterator(); dump_queue(iterator); From 130000e068ec68a10fc632ee4ffc8a5fb563159b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 01:37:00 +0200 Subject: [PATCH 032/273] kernel: Dump scheduler specific thread data --- src/system/kernel/scheduler/scheduler_affine.cpp | 3 ++- src/system/kernel/scheduler/scheduler_simple.cpp | 15 ++++++++++++++- .../kernel/scheduler/scheduler_simple_smp.cpp | 3 ++- src/system/kernel/thread.cpp | 4 ++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index fde10f0d59..471b53ef7c 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -547,7 +547,8 @@ static scheduler_ops kAffineOps = { affine_on_thread_create, affine_on_thread_init, affine_on_thread_destroy, - affine_start + affine_start, + NULL }; diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 40a789aa98..fe359299fd 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -120,6 +120,18 @@ dump_run_queue(int argc, char** argv) } +static void +simple_dump_thread_data(scheduler_thread_data* schedulerThreadData) +{ + kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", + schedulerThreadData->priority_penalty); + kprintf("\tforced_yield_count:\t%" B_PRId32 "\n", + schedulerThreadData->forced_yield_count); + kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", + schedulerThreadData->stolen_time); +} + + static inline int32 simple_get_effective_priority(Thread* thread) { @@ -563,7 +575,8 @@ static scheduler_ops kSimpleOps = { simple_on_thread_create, simple_on_thread_init, simple_on_thread_destroy, - simple_start + simple_start, + simple_dump_thread_data }; diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp index d30ce9c3a7..e596014743 100644 --- a/src/system/kernel/scheduler/scheduler_simple_smp.cpp +++ b/src/system/kernel/scheduler/scheduler_simple_smp.cpp @@ -462,7 +462,8 @@ static scheduler_ops kSimpleSMPOps = { on_thread_create, on_thread_init, on_thread_destroy, - start + start, + NULL }; diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 781720c1c6..66972e70bf 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1786,6 +1786,10 @@ _dump_thread_info(Thread *thread, bool shortInfo) kprintf("flags: 0x%" B_PRIx32 "\n", thread->flags); kprintf("architecture dependant section:\n"); arch_thread_dump_info(&thread->arch_info); + if (gScheduler->dump_thread_data != NULL) { + kprintf("scheduler data:\n"); + gScheduler->dump_thread_data(thread->scheduler_data); + } } From 879ceb60d824345ff90f2f1d6a1e22a086dbe29f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 01:45:07 +0200 Subject: [PATCH 033/273] kernel: Remove suporfluous casts --- .../kernel/scheduler/scheduler_simple.cpp | 49 +++++-------------- 1 file changed, 13 insertions(+), 36 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index fe359299fd..54bb11007a 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -93,12 +93,9 @@ dump_queue(SimpleRunQueue::ConstIterator& iterator) kprintf("thread id priority penalty name\n"); while (iterator.HasNext()) { Thread* thread = iterator.Next(); - scheduler_thread_data* schedulerThreadData - = reinterpret_cast( - thread->scheduler_data); kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", thread, thread->id, thread->priority, - schedulerThreadData->priority_penalty, thread->name); + thread->scheduler_data->priority_penalty, thread->name); } } } @@ -140,11 +137,8 @@ simple_get_effective_priority(Thread* thread) if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) return thread->priority; - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); - int32 effectivePriority = thread->priority; - effectivePriority -= schedulerThreadData->priority_penalty; + effectivePriority -= thread->scheduler_data->priority_penalty; ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); @@ -168,13 +162,10 @@ simple_should_force_yield(Thread* thread) if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) return false; - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); - const int kYieldFrequency = 1 << (min_c(thread->priority, 25) / 5 + 1); - return schedulerThreadData->forced_yield_count != 0 - && schedulerThreadData->forced_yield_count % kYieldFrequency == 0; + return thread->scheduler_data->forced_yield_count != 0 + && thread->scheduler_data->forced_yield_count % kYieldFrequency == 0; } @@ -188,8 +179,7 @@ simple_increase_penalty(Thread* thread) TRACE("increasing thread %ld penalty\n", thread->id); - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; int32 oldPenalty = schedulerThreadData->priority_penalty++; ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); @@ -205,8 +195,7 @@ simple_increase_penalty(Thread* thread) static inline void simple_cancel_penalty(Thread* thread) { - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); @@ -220,8 +209,7 @@ simple_enqueue(Thread* thread, bool newOne) { thread->state = thread->next_state = B_THREAD_READY; - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; if (newOne && hasSlept > kThreadQuantum) @@ -248,11 +236,7 @@ simple_enqueue(Thread* thread, bool newOne) Thread* currentThread = thread_get_current_thread(); if (newOne && threadPriority > currentThread->priority) { - scheduler_thread_data* schedulerCurrentThreadData - = reinterpret_cast( - currentThread->scheduler_data); - - schedulerCurrentThreadData->lost_cpu = true; + currentThread->scheduler_data->lost_cpu = true; gCPU[0].invoke_scheduler = true; gCPU[0].invoke_scheduler_if_idle = false; } @@ -325,10 +309,8 @@ reschedule_event(timer* /* unused */) // This function is called as a result of the timer event set by the // scheduler. Make sure the reschedule() is invoked. Thread* thread= thread_get_current_thread(); - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); - schedulerThreadData->lost_cpu = true; + thread->scheduler_data->lost_cpu = true; thread->cpu->invoke_scheduler = true; thread->cpu->invoke_scheduler_if_idle = false; thread->cpu->preempted = 1; @@ -339,8 +321,7 @@ reschedule_event(timer* /* unused */) static inline bool simple_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) { - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; if (hasYielded) { schedulerThreadData->time_left = 0; @@ -364,8 +345,7 @@ simple_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) static inline bigtime_t simple_compute_quantum(Thread* thread) { - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; bigtime_t quantum; if (schedulerThreadData->time_left != 0) @@ -434,8 +414,7 @@ simple_reschedule(void) TRACE("reschedule(): current thread = %ld\n", oldThread->id); oldThread->state = oldThread->next_state; - scheduler_thread_data* schedulerOldThreadData - = reinterpret_cast(oldThread->scheduler_data); + scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; switch (oldThread->next_state) { case B_THREAD_RUNNING: @@ -542,9 +521,7 @@ simple_on_thread_create(Thread* thread, bool idleThread) static void simple_on_thread_init(Thread* thread) { - scheduler_thread_data* schedulerThreadData - = reinterpret_cast(thread->scheduler_data); - schedulerThreadData->Init(); + thread->scheduler_data->Init(); } From 3e91b082c8b33ef8b8889e7d7693c15eb8f2d9a0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 02:07:08 +0200 Subject: [PATCH 034/273] libroot: Do not rely on thread_yield() --- src/system/libroot/posix/malloc/Jamfile | 2 +- .../libroot/posix/malloc/arch-specific.cpp | 26 +++---------------- .../libroot/posix/malloc/arch-specific.h | 6 ++++- src/system/libroot/posix/malloc/wrapper.cpp | 2 +- 4 files changed, 10 insertions(+), 26 deletions(-) diff --git a/src/system/libroot/posix/malloc/Jamfile b/src/system/libroot/posix/malloc/Jamfile index 3cb65cc7a0..9a14024dd2 100644 --- a/src/system/libroot/posix/malloc/Jamfile +++ b/src/system/libroot/posix/malloc/Jamfile @@ -1,6 +1,6 @@ SubDir HAIKU_TOP src system libroot posix malloc ; -UsePrivateHeaders libroot ; +UsePrivateHeaders libroot shared ; local architectureObject ; for architectureObject in [ MultiArchSubDirSetup ] { diff --git a/src/system/libroot/posix/malloc/arch-specific.cpp b/src/system/libroot/posix/malloc/arch-specific.cpp index 0f16100831..81a39945a8 100644 --- a/src/system/libroot/posix/malloc/arch-specific.cpp +++ b/src/system/libroot/posix/malloc/arch-specific.cpp @@ -41,12 +41,6 @@ void *(*sbrk_hook)(long) = &BPrivate::hoardSbrk; using namespace BPrivate; -// How many iterations we spin waiting for a lock. -enum { SPIN_LIMIT = 50 }; - -// The values of a user-level lock. -enum { UNLOCKED = 0, LOCKED = 1 }; - struct free_chunk { free_chunk *next; size_t size; @@ -368,35 +362,21 @@ hoardUnsbrk(void *ptr, long size) void hoardLockInit(hoardLockType &lock, const char *name) { - lock = UNLOCKED; + mutex_init(&lock, name); } void hoardLock(hoardLockType &lock) { - // A yielding lock (with an initial spin). - while (true) { - int32 i = 0; - while (i < SPIN_LIMIT) { - if (atomic_test_and_set(&lock, LOCKED, UNLOCKED) == UNLOCKED) { - // We got the lock. - return; - } - i++; - } - - // The lock is still being held by someone else. - // Give up our quantum. - hoardYield(); - } + mutex_lock(&lock); } void hoardUnlock(hoardLockType &lock) { - atomic_set(&lock, UNLOCKED); + mutex_unlock(&lock); } diff --git a/src/system/libroot/posix/malloc/arch-specific.h b/src/system/libroot/posix/malloc/arch-specific.h index bffd92310f..88760c2e38 100644 --- a/src/system/libroot/posix/malloc/arch-specific.h +++ b/src/system/libroot/posix/malloc/arch-specific.h @@ -27,8 +27,12 @@ #include #include +#include -typedef int32 hoardLockType; + +// TODO: some kind of adaptive mutex (i.e. trying to spin for a while before +// may be a better choice +typedef mutex hoardLockType; namespace BPrivate { diff --git a/src/system/libroot/posix/malloc/wrapper.cpp b/src/system/libroot/posix/malloc/wrapper.cpp index 2af067fa5b..294901d13c 100644 --- a/src/system/libroot/posix/malloc/wrapper.cpp +++ b/src/system/libroot/posix/malloc/wrapper.cpp @@ -46,7 +46,7 @@ using namespace BPrivate; #if HEAP_LEAK_CHECK static block* sUsedList = NULL; -static hoardLockType sUsedLock = 0; +static hoardLockType sUsedLock = MUTEX_INITIALIZER(""); /*! From a2bdd2842f5f94a8229e61880b57e9314c608e96 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 02:08:49 +0200 Subject: [PATCH 035/273] kernel: Add scheduler_op for dumping thread data --- headers/private/kernel/kscheduler.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index f106ac86c1..18fd30cca3 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -67,6 +67,10 @@ struct scheduler_ops { lock. */ void (*start)(void); + + /*! Dumps scheduler specific thread information. + */ + void (*dump_thread_data)(struct scheduler_thread_data* scheduler_data); }; extern struct scheduler_ops* gScheduler; From 24dbeeddb2990d54291cef7d9b6c3a32034669e7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 02:25:21 +0200 Subject: [PATCH 036/273] kernel: Give longer time slice to lower priority threads --- .../kernel/scheduler/scheduler_simple.cpp | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 54bb11007a..b428863cdb 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -40,7 +40,7 @@ #endif -const bigtime_t kThreadQuantum = 3000; +const bigtime_t kThreadQuantum = 1000; // The run queue. Holds the threads ready to run ordered by priority. @@ -342,6 +342,36 @@ simple_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) } +static inline bigtime_t +simple_quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, + int32 maxPriority, int32 minPriority, int32 priority) +{ + ASSERT(priority <= maxPriority); + ASSERT(priority >= minPriority); + + bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); + result /= maxPriority - minPriority; + return maxQuantum - result; +} + + +static inline bigtime_t +simple_get_base_quantum(Thread* thread) +{ + int32 priority = simple_get_effective_priority(thread); + + if (priority >= B_URGENT_DISPLAY_PRIORITY) + return kThreadQuantum; + if (priority > B_NORMAL_PRIORITY) { + return simple_quantum_linear_interpolation(kThreadQuantum * 4, + kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, + priority); + } + return simple_quantum_linear_interpolation(kThreadQuantum * 64, + kThreadQuantum * 4, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); +} + + static inline bigtime_t simple_compute_quantum(Thread* thread) { @@ -351,7 +381,7 @@ simple_compute_quantum(Thread* thread) if (schedulerThreadData->time_left != 0) quantum = schedulerThreadData->time_left; else - quantum = kThreadQuantum; + quantum = simple_get_base_quantum(thread); quantum += schedulerThreadData->stolen_time; schedulerThreadData->stolen_time = 0; From 4ade765cabf092840feb3771fdde2b37deaf5f27 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 04:26:09 +0200 Subject: [PATCH 037/273] kernel/util: Add heap implementation --- headers/private/kernel/util/Heap.h | 325 +++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 headers/private/kernel/util/Heap.h diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h new file mode 100644 index 0000000000..a39e236ef1 --- /dev/null +++ b/headers/private/kernel/util/Heap.h @@ -0,0 +1,325 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ +#ifndef KERNEL_UTIL_HEAP_H +#define KERNEL_UTIL_HEAP_H + + +#include + +#include + + +template +struct HeapLink { + HeapLink(); + + int fIndex; + Key fKey; +}; + +template +class HeapLinkImpl { +private: + typedef HeapLink Link; + +public: + inline Link* GetHeapLink(); + +private: + Link fHeapLink; +}; + +template +class HeapStandardGetLink { +private: + typedef HeapLink Link; + +public: + inline Link* operator()(Element* element) const; +}; + +template Element::*LinkMember> +class HeapMemberGetLink { +private: + typedef HeapLink Link; + +public: + inline Link* operator()(Element* element) const; +}; + +template +class HeapLesserCompare { +public: + inline bool operator()(Key a, Key b); +}; + +template +class HeapGreaterCompare { +public: + inline bool operator()(Key a, Key b); +}; + +#define HEAP_TEMPLATE_LIST \ + template +#define HEAP_CLASS_NAME Heap + +template, + typename GetLink = HeapStandardGetLink > +class Heap { +public: + Heap(); + ~Heap(); + + inline Element* PeekRoot(); + + inline void ModifyKey(Element* element, Key newKey); + + inline void RemoveRoot(); + inline status_t Insert(Element* element, Key key); + +private: + status_t _GrowHeap(); + + void _MoveUp(HeapLink* link); + void _MoveDown(HeapLink* link); + + Element** fElements; + int fLastElement; + int fSize; + + Compare sCompare; + GetLink sGetLink; + +}; + + +#if KDEBUG +template +HeapLink::HeapLink() + : + fIndex(-1) +{ +} +#else +template +HeapLink::HeapLink() +{ +} +#endif + + +template +HeapLink* +HeapLinkImpl::GetHeapLink() +{ + return &fHeapLink; +} + + +template +HeapLink* +HeapStandardGetLink::operator()(Element* element) const +{ + return element->GetHeapLink(); +} + + +template Element::*LinkMember> +HeapLink* +HeapMemberGetLink::operator()(Element* element) const +{ + return &(element->*LinkMember); +} + + +template +bool +HeapLesserCompare::operator()(Key a, Key b) +{ + return a < b; +} + + +template +bool +HeapGreaterCompare::operator()(Key a, Key b) +{ + return a > b; +} + + +HEAP_TEMPLATE_LIST +HEAP_CLASS_NAME::Heap() + : + fElements(NULL), + fLastElement(0), + fSize(0) +{ +} + + +HEAP_TEMPLATE_LIST +HEAP_CLASS_NAME::~Heap() +{ + free(fElements); +} + + +HEAP_TEMPLATE_LIST +Element* +HEAP_CLASS_NAME::PeekRoot() +{ + if (fLastElement > 0) + return fElements[0]; + return NULL; +} + + +HEAP_TEMPLATE_LIST +void +HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey) +{ + HeapLink link = sGetLink(element); + + ASSERT(link->fIndex >= 0 && link->fIndex < fLastElement); + Key oldKey = link->fKey; + link->fKey = newKey; + + if (sCompare(newKey, oldKey)) + _MoveUp(link); + else + _MoveDown(link); +} + + +HEAP_TEMPLATE_LIST +void +HEAP_CLASS_NAME::RemoveRoot() +{ + ASSERT(fLastElement > 0); + +#if KDEBUG + Element* element = PeekRoot(); + HeapLink link = sGetLink(element); + link->fIndex = -1; +#endif + + fLastElement--; + if (fLastElement > 0) { + Element* lastElement = fElements[fLastElement]; + fElements[0] = lastElement; + sGetLink(lastElement)->fIndex = 0; + _MoveDown(sGetLink(lastElement)); + } +} + + +HEAP_TEMPLATE_LIST +status_t +HEAP_CLASS_NAME::Insert(Element* element, Key key) +{ + if (fLastElement == fSize) { + status_t result = _GrowHeap(); + if (result != B_OK) + return result; + } + + ASSERT(fLastElement != fSize); + + HeapLink* link = sGetLink(element); + + fElements[fLastElement] = element; + link->fIndex = fLastElement++; + link->fKey = key; + _MoveUp(link); +} + + +HEAP_TEMPLATE_LIST +status_t +HEAP_CLASS_NAME::_GrowHeap() +{ + int newSize = max_c(fSize * 2, 4); + + size_t arraySize = newSize * sizeof(Element*); + Element** newBuffer + = reinterpret_cast(realloc(fElements, arraySize)); + if (newBuffer == NULL) + return B_NO_MEMORY; + + fElements = newBuffer; + fSize = newSize; + + return B_OK; +} + + +HEAP_TEMPLATE_LIST +void +HEAP_CLASS_NAME::_MoveUp(HeapLink* link) +{ + int i = link->fIndex; + while (true) { + int parent = (link->fIndex - 1) / 2; + if (link->fIndex > 0 + && sCompare(link->fKey, sGetLink(fElements[parent])->fKey)) { + + sGetLink(fElements[parent])->fIndex = link->fIndex; + + Element* element = fElements[link->fIndex]; + fElements[link->fIndex] = fElements[parent]; + fElements[parent] = element; + + link->fIndex = parent; + } else + break; + } +} + + +HEAP_TEMPLATE_LIST +void +HEAP_CLASS_NAME::_MoveDown(HeapLink* link) +{ + int current; + + while (true) { + current = link->fIndex; + + int child = 2 * link->fIndex + 1; + if (child < fLastElement + && sCompare(sGetLink(fElements[child])->fKey, link->fKey)) { + current = child; + } + + child = 2 * link->fIndex + 2; + if (child < fLastElement + && sCompare(sGetLink(fElements[child])->fKey, + sGetLink(fElements[current])->fKey)) { + current = child; + } + + if (link->fIndex == current) + break; + + sGetLink(fElements[current])->fIndex = link->fIndex; + + Element* element = fElements[link->fIndex]; + fElements[link->fIndex] = fElements[current]; + fElements[current] = element; + + link->fIndex = current; + } +} + + +#endif // KERNEL_UTIL_HEAP_H + From fee8009184189f5aa56d759f3f9974313013fbbc Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 20:13:47 +0200 Subject: [PATCH 038/273] kernel: Add another penalty for CPU bound threads Each thread has its minimal priority that depends on the static priority. However, it is still able to starve threads with even lower priority (e.g. CPU bound threads with lower static priority). To prevent this another penalty is introduced. When the minimal priority is reached penalty (count mod minimal_priority) is added, where count is the number of time slices since the thread reached its minimal priority. This prevents starvation of lower priorirt threads (since all CPU bound threads may have their priority temporaily reduced to 1) but preserves relation between static priorities - when there are two CPU bound threads the one with higher static priority would get more CPU time. --- headers/private/kernel/kscheduler.h | 2 +- .../kernel/scheduler/scheduler_simple.cpp | 46 +++++++++++++++++-- src/system/kernel/thread.cpp | 2 +- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index 18fd30cca3..2c399b1b50 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -70,7 +70,7 @@ struct scheduler_ops { /*! Dumps scheduler specific thread information. */ - void (*dump_thread_data)(struct scheduler_thread_data* scheduler_data); + void (*dump_thread_data)(Thread* thread); }; extern struct scheduler_ops* gScheduler; diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index b428863cdb..01cbb21121 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -55,6 +55,7 @@ struct scheduler_thread_data { void Init(); int32 priority_penalty; + int32 additional_penalty; int32 forced_yield_count; bool lost_cpu; @@ -72,6 +73,7 @@ void scheduler_thread_data::Init() { priority_penalty = 0; + additional_penalty = 0; forced_yield_count = 0; time_left = 0; @@ -84,6 +86,28 @@ scheduler_thread_data::Init() } +static inline int +simple_get_minimal_priority(Thread* thread) +{ + return min_c(thread->priority, 25) / 5; +} + + +static inline int32 +simple_get_thread_penalty(Thread* thread) +{ + int32 penalty = thread->scheduler_data->priority_penalty; + + const int kMinimalPriority = simple_get_minimal_priority(thread); + if (kMinimalPriority > 0) { + penalty + += thread->scheduler_data->additional_penalty % kMinimalPriority; + } + + return penalty; + +} + static inline void dump_queue(SimpleRunQueue::ConstIterator& iterator) { @@ -95,7 +119,7 @@ dump_queue(SimpleRunQueue::ConstIterator& iterator) Thread* thread = iterator.Next(); kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", thread, thread->id, thread->priority, - thread->scheduler_data->priority_penalty, thread->name); + simple_get_thread_penalty(thread), thread->name); } } } @@ -118,10 +142,21 @@ dump_run_queue(int argc, char** argv) static void -simple_dump_thread_data(scheduler_thread_data* schedulerThreadData) +simple_dump_thread_data(Thread* thread) { + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", schedulerThreadData->priority_penalty); + + int32 additionalPenalty = 0; + const int kMinimalPriority = simple_get_minimal_priority(thread); + if (kMinimalPriority > 0) { + additionalPenalty + = schedulerThreadData->additional_penalty % kMinimalPriority; + } + kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", + additionalPenalty, schedulerThreadData->additional_penalty); kprintf("\tforced_yield_count:\t%" B_PRId32 "\n", schedulerThreadData->forced_yield_count); kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", @@ -138,7 +173,7 @@ simple_get_effective_priority(Thread* thread) return thread->priority; int32 effectivePriority = thread->priority; - effectivePriority -= thread->scheduler_data->priority_penalty; + effectivePriority -= simple_get_thread_penalty(thread); ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); @@ -183,11 +218,11 @@ simple_increase_penalty(Thread* thread) int32 oldPenalty = schedulerThreadData->priority_penalty++; ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - const int kMinimalPriority - = min_c(thread->priority, 25) / 5; + const int kMinimalPriority = simple_get_minimal_priority(thread); if (thread->priority - oldPenalty <= kMinimalPriority) { schedulerThreadData->priority_penalty = oldPenalty; schedulerThreadData->forced_yield_count++; + schedulerThreadData->additional_penalty++; } } @@ -200,6 +235,7 @@ simple_cancel_penalty(Thread* thread) if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); schedulerThreadData->priority_penalty = 0; + schedulerThreadData->additional_penalty = 0; schedulerThreadData->forced_yield_count = 0; } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 66972e70bf..c069f2c800 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1788,7 +1788,7 @@ _dump_thread_info(Thread *thread, bool shortInfo) arch_thread_dump_info(&thread->arch_info); if (gScheduler->dump_thread_data != NULL) { kprintf("scheduler data:\n"); - gScheduler->dump_thread_data(thread->scheduler_data); + gScheduler->dump_thread_data(thread); } } From 29e65827fd93f67acbebcdbbe1f233b004a48e18 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 20:42:34 +0200 Subject: [PATCH 039/273] kernel: Remove possibility to yield to all threads Kernel support for yielding to all (including lower priority) threads has been removed. POSIX sched_yield() remains unchanged. If a thread really needs to yield to everyone it can reduce its priority to the lowest possible and then yield (it will then need to manually return to its prvious priority upon continuing). --- headers/private/kernel/thread.h | 2 +- headers/private/kernel/thread_types.h | 1 - .../kernel/bus_managers/random/yarrow_rng.cpp | 2 +- src/system/kernel/arch/x86/arch_cpu.cpp | 2 +- .../kernel/scheduler/scheduler_simple.cpp | 83 +------------------ src/system/kernel/thread.cpp | 12 +-- src/system/kernel/vm/vm_page.cpp | 2 +- 7 files changed, 12 insertions(+), 92 deletions(-) diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 8d1287e172..5ea3f7cbb3 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -86,7 +86,7 @@ void thread_reset_for_exec(void); status_t thread_init(struct kernel_args *args); status_t thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum); -void thread_yield(bool force); +void thread_yield(void); void thread_exit(void); int32 thread_max_threads(void); diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 23e10e9138..b182032bd1 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -444,7 +444,6 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, bool in_kernel; // protected by time_lock, only written by // this thread bool has_yielded; // protected by scheduler lock - bool has_fully_yielded; // protected by scheduler lock struct scheduler_thread_data* scheduler_data; // protected by scheduler lock struct user_thread* user_thread; // write-protected by fLock, only diff --git a/src/add-ons/kernel/bus_managers/random/yarrow_rng.cpp b/src/add-ons/kernel/bus_managers/random/yarrow_rng.cpp index bab1123d12..48e70fdd8f 100644 --- a/src/add-ons/kernel/bus_managers/random/yarrow_rng.cpp +++ b/src/add-ons/kernel/bus_managers/random/yarrow_rng.cpp @@ -158,7 +158,7 @@ reseed(ch_randgen *prandgen, const uint32 initTimes) // random seed on startup by reading from /dev/urandom, perl // programs are all but unusable when at least one other thread // hogs the CPU. - thread_yield(false); + thread_yield(); // TODO: Introduce a clock_counter() function that directly returns // the value of the hardware clock counter. This will be cheaper diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index ed1e7ebb82..7ea90bf672 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -155,7 +155,7 @@ acpi_shutdown(bool rebootSystem) _user_set_cpu_enabled(cpu, false); } // TODO: must not be called from the idle thread! - thread_yield(true); + thread_yield(); status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0); if (status == B_OK) { diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 01cbb21121..dba594a434 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -46,8 +46,6 @@ const bigtime_t kThreadQuantum = 1000; // The run queue. Holds the threads ready to run ordered by priority. typedef RunQueue SimpleRunQueue; static SimpleRunQueue* sRunQueue; -static SimpleRunQueue* sExpiredQueue; -static int32 sYieldedThreadPriority = -1; struct scheduler_thread_data { @@ -56,7 +54,6 @@ struct scheduler_thread_data { int32 priority_penalty; int32 additional_penalty; - int32 forced_yield_count; bool lost_cpu; bool cpu_bound; @@ -74,7 +71,6 @@ scheduler_thread_data::Init() { priority_penalty = 0; additional_penalty = 0; - forced_yield_count = 0; time_left = 0; stolen_time = 0; @@ -133,10 +129,6 @@ dump_run_queue(int argc, char** argv) iterator = sRunQueue->GetConstIterator(); dump_queue(iterator); - kprintf("\nExpired run queue:\n"); - iterator = sExpiredQueue->GetConstIterator(); - dump_queue(iterator); - return 0; } @@ -157,8 +149,6 @@ simple_dump_thread_data(Thread* thread) } kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", additionalPenalty, schedulerThreadData->additional_penalty); - kprintf("\tforced_yield_count:\t%" B_PRId32 "\n", - schedulerThreadData->forced_yield_count); kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", schedulerThreadData->stolen_time); } @@ -182,28 +172,6 @@ simple_get_effective_priority(Thread* thread) } -static inline void -simple_yield(Thread* thread) -{ - TRACE("thread %ld yielded\n", thread->id); - int32 effectivePriority = simple_get_effective_priority(thread); - sYieldedThreadPriority = max_c(sYieldedThreadPriority, effectivePriority); -} - - -static inline bool -simple_should_force_yield(Thread* thread) -{ - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return false; - - const int kYieldFrequency = 1 << (min_c(thread->priority, 25) / 5 + 1); - - return thread->scheduler_data->forced_yield_count != 0 - && thread->scheduler_data->forced_yield_count % kYieldFrequency == 0; -} - - static inline void simple_increase_penalty(Thread* thread) { @@ -221,7 +189,6 @@ simple_increase_penalty(Thread* thread) const int kMinimalPriority = simple_get_minimal_priority(thread); if (thread->priority - oldPenalty <= kMinimalPriority) { schedulerThreadData->priority_penalty = oldPenalty; - schedulerThreadData->forced_yield_count++; schedulerThreadData->additional_penalty++; } } @@ -236,7 +203,6 @@ simple_cancel_penalty(Thread* thread) TRACE("cancelling thread %ld penalty\n", thread->id); schedulerThreadData->priority_penalty = 0; schedulerThreadData->additional_penalty = 0; - schedulerThreadData->forced_yield_count = 0; } @@ -251,16 +217,10 @@ simple_enqueue(Thread* thread, bool newOne) if (newOne && hasSlept > kThreadQuantum) simple_cancel_penalty(thread); - if (simple_should_force_yield(thread)) - simple_yield(thread); - int32 threadPriority = simple_get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); - if (threadPriority <= sYieldedThreadPriority) - sExpiredQueue->PushBack(thread, threadPriority); - else - sRunQueue->PushBack(thread, threadPriority); + sRunQueue->PushBack(thread, threadPriority); schedulerThreadData->cpu_bound = true; schedulerThreadData->time_left = 0; @@ -429,35 +389,6 @@ simple_compute_quantum(Thread* thread) } -static inline Thread* -simple_get_next_thread(void) -{ - Thread* thread; - do { - thread = sRunQueue->PeekMaximum(); - - if (sYieldedThreadPriority >= 0 && thread != NULL - && thread_is_idle_thread(thread)) { - sRunQueue->Remove(thread); - simple_enqueue_in_run_queue(thread); - continue; - } - - break; - } while (true); - if (thread == NULL && sYieldedThreadPriority >= 0) { - SimpleRunQueue* temp = sRunQueue; - sRunQueue = sExpiredQueue; - sExpiredQueue = temp; - sYieldedThreadPriority = -1; - - thread = sRunQueue->PeekMaximum(); - } - - return thread; -} - - /*! Runs the scheduler. Note: expects thread spinlock to be held */ @@ -493,9 +424,6 @@ simple_reschedule(void) if (schedulerOldThreadData->cpu_bound) simple_increase_penalty(oldThread); - if (oldThread->has_fully_yielded) - simple_yield(oldThread); - TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, simple_get_effective_priority(oldThread)); simple_enqueue(oldThread, false); @@ -521,11 +449,10 @@ simple_reschedule(void) } oldThread->has_yielded = false; - oldThread->has_fully_yielded = false; schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - Thread* nextThread = simple_get_next_thread(); + Thread* nextThread = sRunQueue->PeekMaximum(); if (!nextThread) panic("reschedule(): run queues are empty!\n"); sRunQueue->Remove(nextThread); @@ -634,11 +561,6 @@ scheduler_simple_init() return B_NO_MEMORY; ObjectDeleter runQueueDeleter(sRunQueue); - sExpiredQueue = new(std::nothrow) SimpleRunQueue; - if (sExpiredQueue == NULL) - return B_NO_MEMORY; - ObjectDeleter expiredQueueDeleter(sExpiredQueue); - status_t result = sRunQueue->GetInitStatus(); if (result != B_OK) return result; @@ -649,6 +571,5 @@ scheduler_simple_init() "List threads in run queue", "\nLists threads in run queue", 0); runQueueDeleter.Detach(); - expiredQueueDeleter.Detach(); return B_OK; } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index c069f2c800..e3c45319c0 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -179,7 +179,6 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) signal_stack_enabled(false), in_kernel(true), has_yielded(false), - has_fully_yielded(false), user_thread(NULL), fault_handler(0), page_faults_allowed(1), @@ -2440,14 +2439,15 @@ peek_next_thread_id() /*! Yield the CPU to other threads. - If \a force is \c true, the thread will almost guaranteedly be unscheduled. - If \c false, it will continue to run, if there's no other thread in ready + Thread will continue to run, if there's no other thread in ready state, and if it has a higher priority than the other ready threads, it still has a good chance to continue. */ void -thread_yield(bool force) +thread_yield(void) { + // Yielding is for being nice, not for making things work. +#if !KDEBUG Thread *thread = thread_get_current_thread(); if (thread == NULL) return; @@ -2455,8 +2455,8 @@ thread_yield(bool force) InterruptsSpinLocker _(gSchedulerLock); thread->has_yielded = true; - thread->has_fully_yielded = force; scheduler_reschedule(); +#endif } @@ -3507,7 +3507,7 @@ _user_snooze_etc(bigtime_t timeout, int timebase, uint32 flags, void _user_thread_yield(void) { - thread_yield(false); + thread_yield(); } diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index ee20257164..35d1a8bfee 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -2390,7 +2390,7 @@ page_writer(void* /*unused*/) if (cache->AcquireUnreferencedStoreRef() != B_OK) { DEBUG_PAGE_ACCESS_END(page); cacheLocker.Unlock(); - thread_yield(true); + thread_yield(); continue; } From 298314fe4b1f592c5b2caf3f06a1e8a452329396 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 9 Oct 2013 20:57:42 +0200 Subject: [PATCH 040/273] libroot: Update sched_get_priority_{max, min}() SCHED_RR is a real-time scheduling policy. SCHED_FIFO and SCHED_SPORADIC are not supported (at least for now). --- src/system/libroot/posix/scheduler.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/system/libroot/posix/scheduler.cpp b/src/system/libroot/posix/scheduler.cpp index f56543d023..a77f82a3ed 100644 --- a/src/system/libroot/posix/scheduler.cpp +++ b/src/system/libroot/posix/scheduler.cpp @@ -26,9 +26,9 @@ int sched_get_priority_min(int policy) { switch (policy) { - case SCHED_FIFO: case SCHED_RR: - case SCHED_SPORADIC: + return B_FIRST_REAL_TIME_PRIORITY; + case SCHED_OTHER: return B_LOW_PRIORITY; @@ -43,9 +43,9 @@ int sched_get_priority_max(int policy) { switch (policy) { - case SCHED_FIFO: case SCHED_RR: - case SCHED_SPORADIC: + return B_REAL_TIME_PRIORITY; + case SCHED_OTHER: return B_URGENT_DISPLAY_PRIORITY; From f20ad54be26d164713bd06dab0435cee985561ef Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 15 Oct 2013 00:29:04 +0200 Subject: [PATCH 041/273] kernel: Add support for SMP systems to simple scheduler In SMP systems simple scheduler will be used only when all logical processors share all levels of cache and the number of CPUs is low. In such systems we do not have to care about cache affinity and the contention on the lock protecting shared run queue is low. Single run queue makes load balancing very simple. --- headers/private/kernel/util/Heap.h | 20 +- src/system/kernel/scheduler/scheduler.cpp | 14 +- .../kernel/scheduler/scheduler_simple.cpp | 178 +++++++++++++++--- src/system/kernel/thread.cpp | 17 +- 4 files changed, 173 insertions(+), 56 deletions(-) diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h index a39e236ef1..cca75c7569 100644 --- a/headers/private/kernel/util/Heap.h +++ b/headers/private/kernel/util/Heap.h @@ -79,6 +79,8 @@ public: inline Element* PeekRoot(); + inline const Key& GetKey(Element* element) const; + inline void ModifyKey(Element* element, Key newKey); inline void RemoveRoot(); @@ -183,11 +185,22 @@ HEAP_CLASS_NAME::PeekRoot() } +HEAP_TEMPLATE_LIST +const Key& +HEAP_CLASS_NAME::GetKey(Element* element) const +{ + HeapLink* link = sGetLink(element); + + ASSERT(link->fIndex >= 0 && link->fIndex < fLastElement); + return link->fKey; +} + + HEAP_TEMPLATE_LIST void HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey) { - HeapLink link = sGetLink(element); + HeapLink* link = sGetLink(element); ASSERT(link->fIndex >= 0 && link->fIndex < fLastElement); Key oldKey = link->fKey; @@ -208,7 +221,7 @@ HEAP_CLASS_NAME::RemoveRoot() #if KDEBUG Element* element = PeekRoot(); - HeapLink link = sGetLink(element); + HeapLink* link = sGetLink(element); link->fIndex = -1; #endif @@ -240,6 +253,8 @@ HEAP_CLASS_NAME::Insert(Element* element, Key key) link->fIndex = fLastElement++; link->fKey = key; _MoveUp(link); + + return B_OK; } @@ -266,7 +281,6 @@ HEAP_TEMPLATE_LIST void HEAP_CLASS_NAME::_MoveUp(HeapLink* link) { - int i = link->fIndex; while (true) { int parent = (link->fIndex - 1) / 2; if (link->fIndex > 0 diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index a8ed35a985..7087689f86 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -68,18 +68,12 @@ scheduler_init(void) cpuCount != 1 ? "s" : ""); status_t result; - if (cpuCount > 1) { #if 0 - dprintf("scheduler_init: using affine scheduler\n"); - result = scheduler_affine_init(); -#else - dprintf("scheduler_init: using simple SMP scheduler\n"); - result = scheduler_simple_smp_init(); + dprintf("scheduler_init: using affine scheduler\n"); + result = scheduler_affine_init(); #endif - } else { - dprintf("scheduler_init: using simple scheduler\n"); - result = scheduler_simple_init(); - } + dprintf("scheduler_init: using simple scheduler\n"); + result = scheduler_simple_init(); if (result != B_OK) panic("scheduler_init: failed to initialize scheduler\n"); diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index dba594a434..4057752548 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "RunQueue.h" @@ -43,6 +44,14 @@ const bigtime_t kThreadQuantum = 1000; +struct CPUHeapEntry : public HeapLinkImpl { + int32 fCPUNumber; +}; + +static CPUHeapEntry* sCPUEntries; +typedef Heap SimpleCPUHeap; +static SimpleCPUHeap* sCPUHeap; + // The run queue. Holds the threads ready to run ordered by priority. typedef RunQueue SimpleRunQueue; static SimpleRunQueue* sRunQueue; @@ -104,11 +113,30 @@ simple_get_thread_penalty(Thread* thread) } -static inline void + +static inline int32 +simple_get_effective_priority(Thread* thread) +{ + if (thread->priority == B_IDLE_PRIORITY) + return thread->priority; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return thread->priority; + + int32 effectivePriority = thread->priority; + effectivePriority -= simple_get_thread_penalty(thread); + + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return effectivePriority; +} + + +static void dump_queue(SimpleRunQueue::ConstIterator& iterator) { if (!iterator.HasNext()) - kprintf("Queue is empty.\n"); + kprintf("Run queue is empty.\n"); else { kprintf("thread id priority penalty name\n"); while (iterator.HasNext()) { @@ -124,15 +152,38 @@ dump_queue(SimpleRunQueue::ConstIterator& iterator) static int dump_run_queue(int argc, char** argv) { - SimpleRunQueue::ConstIterator iterator; - kprintf("Current run queue:\n"); - iterator = sRunQueue->GetConstIterator(); + SimpleRunQueue::ConstIterator iterator = sRunQueue->GetConstIterator(); dump_queue(iterator); return 0; } +static int +dump_cpu_heap(int argc, char** argv) +{ + kprintf("\ncpu priority actual priority\n"); + CPUHeapEntry* entry = sCPUHeap->PeekRoot(); + while (entry) { + int32 cpu = entry->fCPUNumber; + kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, + sCPUHeap->GetKey(entry), + simple_get_effective_priority(gCPU[cpu].running_thread)); + + sCPUHeap->RemoveRoot(); + entry = sCPUHeap->PeekRoot(); + } + + int32 cpuCount = smp_get_num_cpus(); + for (int i = 0; i < cpuCount; i++) { + sCPUHeap->Insert(&sCPUEntries[i], + simple_get_effective_priority(gCPU[i].running_thread)); + } + + return 0; +} + + static void simple_dump_thread_data(Thread* thread) { @@ -154,24 +205,6 @@ simple_dump_thread_data(Thread* thread) } -static inline int32 -simple_get_effective_priority(Thread* thread) -{ - if (thread->priority == B_IDLE_PRIORITY) - return thread->priority; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return thread->priority; - - int32 effectivePriority = thread->priority; - effectivePriority -= simple_get_thread_penalty(thread); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; -} - - static inline void simple_increase_penalty(Thread* thread) { @@ -218,6 +251,7 @@ simple_enqueue(Thread* thread, bool newOne) simple_cancel_penalty(thread); int32 threadPriority = simple_get_effective_priority(thread); + T(EnqueueThread(thread, threadPriority)); sRunQueue->PushBack(thread, threadPriority); @@ -230,11 +264,44 @@ simple_enqueue(Thread* thread, bool newOne) NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - Thread* currentThread = thread_get_current_thread(); - if (newOne && threadPriority > currentThread->priority) { - currentThread->scheduler_data->lost_cpu = true; - gCPU[0].invoke_scheduler = true; - gCPU[0].invoke_scheduler_if_idle = false; + // TODO: pinned threads + // TODO: disabled CPUs + CPUHeapEntry* cpuEntry = sCPUHeap->PeekRoot(); + ASSERT(cpuEntry != NULL); + + int32 thisCPU = smp_get_current_cpu(); + int32 targetCPU = cpuEntry->fCPUNumber; + Thread* targetThread = gCPU[targetCPU].running_thread; + int32 targetPriority = simple_get_effective_priority(targetThread); + + ASSERT((targetCPU != thisCPU && targetThread != thread) + || targetCPU == thisCPU); + + int32 currentThreadPriority + = simple_get_effective_priority(thread_get_current_thread()); + if (targetPriority == currentThreadPriority) { + targetCPU = thisCPU; + targetPriority = currentThreadPriority; + } + + TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, + targetPriority); + + if (threadPriority > targetPriority) { + targetThread->scheduler_data->lost_cpu = true; + + // It is possible that another CPU schedules the thread before the + // target CPU. However, since the target CPU is sent an ICI it will + // reschedule anyway and update its heap key to the correct value. + sCPUHeap->ModifyKey(cpuEntry, threadPriority); + + if (targetCPU == smp_get_current_cpu()) { + gCPU[targetCPU].invoke_scheduler = true; + gCPU[targetCPU].invoke_scheduler_if_idle = false; + } else { + smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, + SMP_MSG_FLAG_ASYNC); + } } } @@ -245,6 +312,8 @@ simple_enqueue(Thread* thread, bool newOne) static void simple_enqueue_in_run_queue(Thread* thread) { + TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, + thread->priority); simple_enqueue(thread, true); } @@ -258,7 +327,15 @@ simple_set_thread_priority(Thread* thread, int32 priority) if (priority == thread->priority) return; + TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", + thread->id, priority, thread->priority, + simple_get_effective_priority(thread)); + + if (thread->state == B_THREAD_RUNNING) + sCPUHeap->ModifyKey(&sCPUEntries[thread->cpu->cpu_num], priority); + if (thread->state != B_THREAD_READY) { + simple_cancel_penalty(thread); thread->priority = priority; return; } @@ -408,11 +485,21 @@ simple_reschedule(void) } } - TRACE("reschedule(): current thread = %ld\n", oldThread->id); + int32 thisCPU = smp_get_current_cpu(); + + TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, + oldThread->id); oldThread->state = oldThread->next_state; scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; + // update CPU heap so that old thread would have CPU properly chosen + Thread* nextThread = sRunQueue->PeekMaximum(); + if (nextThread != NULL) { + sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], + simple_get_effective_priority(nextThread)); + } + switch (oldThread->next_state) { case B_THREAD_RUNNING: case B_THREAD_READY: @@ -452,15 +539,20 @@ simple_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - Thread* nextThread = sRunQueue->PeekMaximum(); + nextThread = sRunQueue->PeekMaximum(); if (!nextThread) panic("reschedule(): run queues are empty!\n"); sRunQueue->Remove(nextThread); - TRACE("reschedule(): next thread = %ld\n", nextThread->id); + TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, + nextThread->id); T(ScheduleThread(nextThread, oldThread)); + // update CPU heap + sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], + simple_get_effective_priority(nextThread)); + // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, oldThread, nextThread); @@ -556,6 +648,25 @@ static scheduler_ops kSimpleOps = { status_t scheduler_simple_init() { + int32 cpuCount = smp_get_num_cpus(); + + sCPUHeap = new SimpleCPUHeap; + if (sCPUHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter cpuHeapDeleter(sCPUHeap); + + sCPUEntries = new CPUHeapEntry[cpuCount]; + if (sCPUEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuEntriesDeleter(sCPUEntries); + + for (int i = 0; i < cpuCount; i++) { + sCPUEntries[i].fCPUNumber = i; + status_t result = sCPUHeap->Insert(&sCPUEntries[i], B_IDLE_PRIORITY); + if (result != B_OK) + return result; + } + sRunQueue = new(std::nothrow) SimpleRunQueue; if (sRunQueue == NULL) return B_NO_MEMORY; @@ -569,7 +680,12 @@ scheduler_simple_init() add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", + 0); + cpuHeapDeleter.Detach(); + cpuEntriesDeleter.Detach(); runQueueDeleter.Detach(); return B_OK; } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index e3c45319c0..24ed01d800 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1425,7 +1425,7 @@ make_thread_unreal(int argc, char **argv) continue; if (thread->priority > B_DISPLAY_PRIORITY) { - thread->priority = B_NORMAL_PRIORITY; + scheduler_set_thread_priority(thread, B_NORMAL_PRIORITY); kprintf("thread %" B_PRId32 " made unreal\n", thread->id); } } @@ -1461,7 +1461,7 @@ set_thread_prio(int argc, char **argv) Thread* thread = it.Next();) { if (thread->id != id) continue; - thread->priority = prio; + scheduler_set_thread_priority(thread, prio); kprintf("thread %" B_PRId32 " set to priority %" B_PRId32 "\n", id, prio); found = true; break; @@ -1920,7 +1920,7 @@ thread_exit(void) panic("thread_exit() called with interrupts disabled!\n"); // boost our priority to get this over with - thread->priority = B_URGENT_DISPLAY_PRIORITY; + scheduler_set_thread_priority(thread, B_URGENT_DISPLAY_PRIORITY); if (team != kernelTeam) { // Cancel previously installed alarm timer, if any. Hold the scheduler @@ -3205,15 +3205,8 @@ set_thread_priority(thread_id id, int32 priority) InterruptsSpinLocker schedulerLocker(gSchedulerLock); - if (thread == thread_get_current_thread()) { - // It's ourself, so we know we aren't in the run queue, and we can - // manipulate our structure directly. - oldPriority = thread->priority; - thread->priority = priority; - } else { - oldPriority = thread->priority; - scheduler_set_thread_priority(thread, priority); - } + oldPriority = thread->priority; + scheduler_set_thread_priority(thread, priority); return oldPriority; } From 51d1e9ada0e963b344954f7d6b7e94399197eeaf Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 15 Oct 2013 00:37:19 +0200 Subject: [PATCH 042/273] kernel: Remove scheduler_simple_smp --- src/system/kernel/Jamfile | 1 - src/system/kernel/scheduler/scheduler.cpp | 1 - .../kernel/scheduler/scheduler_simple_smp.cpp | 484 ------------------ .../kernel/scheduler/scheduler_simple_smp.h | 12 - 4 files changed, 498 deletions(-) delete mode 100644 src/system/kernel/scheduler/scheduler_simple_smp.cpp delete mode 100644 src/system/kernel/scheduler/scheduler_simple_smp.h diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 551317d75b..2c0079bdc7 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -65,7 +65,6 @@ KernelMergeObject kernel_core.o : scheduler.cpp scheduler_affine.cpp scheduler_simple.cpp - scheduler_simple_smp.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 7087689f86..ba6a00825f 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -11,7 +11,6 @@ #include "scheduler_affine.h" #include "scheduler_simple.h" -#include "scheduler_simple_smp.h" #include "scheduler_tracing.h" diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.cpp b/src/system/kernel/scheduler/scheduler_simple_smp.cpp deleted file mode 100644 index e596014743..0000000000 --- a/src/system/kernel/scheduler/scheduler_simple_smp.cpp +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. - * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. - * Copyright 2002, Angelo Mottola, a.mottola@libero.it. - * Distributed under the terms of the MIT License. - * - * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. - * Distributed under the terms of the NewOS License. - */ - - -/*! The thread scheduler */ - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "scheduler_common.h" -#include "scheduler_tracing.h" - - -//#define TRACE_SCHEDULER -#ifdef TRACE_SCHEDULER -# define TRACE(x) dprintf_no_syslog x -#else -# define TRACE(x) ; -#endif - - -const bigtime_t kThreadQuantum = 3000; - - -// The run queue. Holds the threads ready to run ordered by priority. -static Thread *sRunQueue = NULL; -static int32 sCPUCount = 1; -static int32 sNextCPUForSelection = 0; - - -static int -dump_run_queue(int argc, char **argv) -{ - Thread *thread; - - thread = sRunQueue; - if (!thread) - kprintf("Run queue is empty!\n"); - else { - kprintf("thread id priority name\n"); - while (thread) { - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %s\n", thread, - thread->id, thread->priority, thread->name); - thread = thread->queue_next; - } - } - - return 0; -} - - -static int32 -select_cpu(int32 currentCPU, Thread* thread, int32& targetPriority) -{ - if (thread->pinned_to_cpu > 0) { - // the thread is pinned to a specific CPU - int32 targetCPU = thread->previous_cpu->cpu_num; - targetPriority = gCPU[targetCPU].running_thread->priority; - return targetCPU; - } - - // Choose the CPU running the lowest priority thread. Favor the current CPU - // as it doesn't require ICI to be notified. - int32 targetCPU = currentCPU; - targetPriority = B_IDLE_PRIORITY; - if (gCPU[currentCPU].disabled) - targetCPU = -1; - else - targetPriority = gCPU[currentCPU].running_thread->priority; - - int32 cpu = sNextCPUForSelection; - for (int32 i = 0; i < sCPUCount; i++, cpu++) { - if (cpu >= sCPUCount) - cpu = 0; - - if (!gCPU[cpu].disabled) { - int32 cpuPriority = gCPU[cpu].running_thread->priority; - if (targetCPU < 0 || cpuPriority < targetPriority) { - targetCPU = cpu; - targetPriority = cpuPriority; - } - } - } - - if (++sNextCPUForSelection >= sCPUCount) - sNextCPUForSelection = 0; - - return targetCPU; -} - - -/*! Enqueues the thread into the run queue. - Note: thread lock must be held when entering this function -*/ -static void -enqueue_in_run_queue(Thread *thread) -{ - thread->state = thread->next_state = B_THREAD_READY; - - Thread *curr, *prev; - for (curr = sRunQueue, prev = NULL; curr - && curr->priority >= thread->priority; - curr = curr->queue_next) { - if (prev) - prev = prev->queue_next; - else - prev = sRunQueue; - } - - T(EnqueueThread(thread, thread->priority)); - - thread->queue_next = curr; - if (prev) - prev->queue_next = thread; - else - sRunQueue = thread; - - if (thread->priority != B_IDLE_PRIORITY) { - // Select a CPU for the thread to run on. It's not certain that the - // thread will actually run on it, but we will notify the CPU to - // preempt the thread it is currently running, if the new thread has - // a higher priority. - int32 currentCPU = smp_get_current_cpu(); - int32 targetPriority; - int32 targetCPU = select_cpu(currentCPU, thread, targetPriority); - - // If the target CPU runs a thread with a lower priority, tell it to - // reschedule. - if (thread->priority > targetPriority) { - if (targetCPU == currentCPU) { - gCPU[targetCPU].invoke_scheduler = true; - gCPU[targetCPU].invoke_scheduler_if_idle = false; - } else { - if (targetPriority == B_IDLE_PRIORITY) { - smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE_IF_IDLE, 0, 0, - 0, NULL, SMP_MSG_FLAG_ASYNC); - } else { - smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, - SMP_MSG_FLAG_ASYNC); - } - } - } - } - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, - thread); -} - - -/*! Sets the priority of a thread. - Note: thread lock must be held when entering this function -*/ -static void -set_thread_priority(Thread *thread, int32 priority) -{ - if (priority == thread->priority) - return; - - if (thread->state != B_THREAD_READY) { - thread->priority = priority; - return; - } - - // The thread is in the run queue. We need to remove it and re-insert it at - // a new position. - - T(RemoveThread(thread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, - thread); - - // find thread in run queue - Thread *item, *prev; - for (item = sRunQueue, prev = NULL; item && item != thread; - item = item->queue_next) { - if (prev) - prev = prev->queue_next; - else - prev = sRunQueue; - } - - ASSERT(item == thread); - - // remove the thread - if (prev) - prev->queue_next = item->queue_next; - else - sRunQueue = item->queue_next; - - // set priority and re-insert - thread->priority = priority; - enqueue_in_run_queue(thread); -} - - -static bigtime_t -estimate_max_scheduling_latency(Thread* thread) -{ - // TODO: This is probably meant to be called periodically to return the - // current estimate depending on the system usage; we return fixed estimates - // per thread priority, though. - - if (thread->priority >= B_REAL_TIME_DISPLAY_PRIORITY) - return kThreadQuantum / 4; - if (thread->priority >= B_DISPLAY_PRIORITY) - return kThreadQuantum; - - return 2 * kThreadQuantum; -} - - -static int32 -reschedule_event(timer *unused) -{ - // This function is called as a result of the timer event set by the - // scheduler. Make sure the reschedule() is invoked. - thread_get_current_thread()->cpu->invoke_scheduler = true; - thread_get_current_thread()->cpu->invoke_scheduler_if_idle = false; - thread_get_current_thread()->cpu->preempted = 1; - return B_HANDLED_INTERRUPT; -} - - -/*! Runs the scheduler. - Note: expects thread spinlock to be held -*/ -static void -reschedule(void) -{ - Thread *oldThread = thread_get_current_thread(); - Thread *nextThread, *prevThread; - - // check whether we're only supposed to reschedule, if the current thread - // is idle - if (oldThread->cpu->invoke_scheduler) { - oldThread->cpu->invoke_scheduler = false; - if (oldThread->cpu->invoke_scheduler_if_idle - && oldThread->priority != B_IDLE_PRIORITY) { - oldThread->cpu->invoke_scheduler_if_idle = false; - return; - } - } - - TRACE(("reschedule(): cpu %ld, cur_thread = %ld\n", smp_get_current_cpu(), - thread_get_current_thread()->id)); - - oldThread->state = oldThread->next_state; - switch (oldThread->next_state) { - case B_THREAD_RUNNING: - case B_THREAD_READY: - TRACE(("enqueueing thread %ld into run q. pri = %ld\n", - oldThread->id, oldThread->priority)); - enqueue_in_run_queue(oldThread); - break; - case B_THREAD_SUSPENDED: - TRACE(("reschedule(): suspending thread %ld\n", oldThread->id)); - break; - case THREAD_STATE_FREE_ON_RESCHED: - break; - default: - TRACE(("not enqueueing thread %ld into run q. next_state = %ld\n", - oldThread->id, oldThread->next_state)); - break; - } - - nextThread = sRunQueue; - prevThread = NULL; - - if (oldThread->cpu->disabled) { - // CPU is disabled - service any threads we may have that are pinned, - // otherwise just select the idle thread - while (nextThread != NULL && nextThread->priority > B_IDLE_PRIORITY) { - if (nextThread->pinned_to_cpu > 0 - && nextThread->previous_cpu == oldThread->cpu) - break; - prevThread = nextThread; - nextThread = nextThread->queue_next; - } - } else { - while (nextThread != NULL) { - // select next thread from the run queue - // TODO: nextThread cannot really be NULL here, so we should be able - // to remove the check, as well as the panic later on. - while (nextThread != NULL - && nextThread->priority > B_IDLE_PRIORITY) { -#if 0 - if (oldThread == nextThread && nextThread->was_yielded) { - // ignore threads that called thread_yield() once - nextThread->was_yielded = false; - prevThread = nextThread; - nextThread = nextThread->queue_next; - } -#endif - - // skip thread, if it doesn't want to run on this CPU - if (nextThread->pinned_to_cpu > 0 - && nextThread->previous_cpu != oldThread->cpu) { - prevThread = nextThread; - nextThread = nextThread->queue_next; - continue; - } - - // always extract real time threads - if (nextThread->priority >= B_FIRST_REAL_TIME_PRIORITY) - break; - - // find next thread with lower priority - Thread *lowerNextThread = nextThread->queue_next; - Thread *lowerPrevThread = nextThread; - int32 priority = nextThread->priority; - - while (lowerNextThread != NULL - && priority == lowerNextThread->priority) { - lowerPrevThread = lowerNextThread; - lowerNextThread = lowerNextThread->queue_next; - } - // never skip last non-idle normal thread - if (lowerNextThread == NULL - || lowerNextThread->priority == B_IDLE_PRIORITY) - break; - - int32 priorityDiff = priority - lowerNextThread->priority; - if (priorityDiff > 15) - break; - - // skip normal threads sometimes - // (twice as probable per priority level) - if ((fast_random_value() >> (15 - priorityDiff)) != 0) - break; - - nextThread = lowerNextThread; - prevThread = lowerPrevThread; - } - - if (nextThread != NULL && nextThread->cpu - && nextThread->cpu->cpu_num != oldThread->cpu->cpu_num) { - panic("thread in run queue that's still running on another CPU!\n"); - // TODO: remove this check completely when we're sure that this - // cannot happen anymore. - prevThread = nextThread; - nextThread = nextThread->queue_next; - continue; - } - - break; - } - } - - if (nextThread == NULL) - panic("reschedule(): run queue is empty!\n"); - - // extract selected thread from the run queue - if (prevThread != NULL) - prevThread->queue_next = nextThread->queue_next; - else - sRunQueue = nextThread->queue_next; - - T(ScheduleThread(nextThread, oldThread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, oldThread, - nextThread); - - nextThread->state = B_THREAD_RUNNING; - nextThread->next_state = B_THREAD_READY; - oldThread->has_yielded = false; - - // track kernel time (user time is tracked in thread_at_kernel_entry()) - scheduler_update_thread_times(oldThread, nextThread); - - // track CPU activity - if (!thread_is_idle_thread(oldThread)) { - oldThread->cpu->active_time - += (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time); - } - - if (!thread_is_idle_thread(nextThread)) { - oldThread->cpu->last_kernel_time = nextThread->kernel_time; - oldThread->cpu->last_user_time = nextThread->user_time; - } - - if (nextThread != oldThread || oldThread->cpu->preempted) { - bigtime_t quantum = kThreadQuantum; // TODO: calculate quantum? - timer* quantumTimer = &oldThread->cpu->quantum_timer; - - if (!oldThread->cpu->preempted) - cancel_timer(quantumTimer); - - oldThread->cpu->preempted = 0; - if (!thread_is_idle_thread(nextThread)) { - add_timer(quantumTimer, &reschedule_event, quantum, - B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); - } - - if (nextThread != oldThread) - scheduler_switch_thread(oldThread, nextThread); - } -} - - -static status_t -on_thread_create(Thread* thread, bool idleThread) -{ - // do nothing - return B_OK; -} - - -static void -on_thread_init(Thread* thread) -{ - // do nothing -} - - -static void -on_thread_destroy(Thread* thread) -{ - // do nothing -} - - -/*! This starts the scheduler. Must be run in the context of the initial idle - thread. Interrupts must be disabled and will be disabled when returning. -*/ -static void -start(void) -{ - SpinLocker schedulerLocker(gSchedulerLock); - - reschedule(); -} - - -static scheduler_ops kSimpleSMPOps = { - enqueue_in_run_queue, - reschedule, - set_thread_priority, - estimate_max_scheduling_latency, - on_thread_create, - on_thread_init, - on_thread_destroy, - start, - NULL -}; - - -// #pragma mark - - - -status_t -scheduler_simple_smp_init() -{ - sCPUCount = smp_get_num_cpus(); - - gScheduler = &kSimpleSMPOps; - - add_debugger_command_etc("run_queue", &dump_run_queue, - "List threads in run queue", "\nLists threads in run queue", 0); - - return B_OK; -} diff --git a/src/system/kernel/scheduler/scheduler_simple_smp.h b/src/system/kernel/scheduler/scheduler_simple_smp.h deleted file mode 100644 index d59a02841f..0000000000 --- a/src/system/kernel/scheduler/scheduler_simple_smp.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. - * Distributed under the terms of the MIT License. - */ -#ifndef KERNEL_SCHEDULER_SIMPLE_SMP_H -#define KERNEL_SCHEDULER_SIMPLE_SMP_H - - -status_t scheduler_simple_smp_init(); - - -#endif // KERNEL_SCHEDULER_SIMPLE_SMP_H From 3de2c5ceec4a4b2cbbe585068b173f860011a527 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 15 Oct 2013 01:42:18 +0200 Subject: [PATCH 043/273] kernel: Add support for pinned threads --- .../kernel/scheduler/scheduler_simple.cpp | 118 +++++++++++++++--- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 4057752548..fc8fe9b9b4 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -55,6 +55,7 @@ static SimpleCPUHeap* sCPUHeap; // The run queue. Holds the threads ready to run ordered by priority. typedef RunQueue SimpleRunQueue; static SimpleRunQueue* sRunQueue; +static SimpleRunQueue* sCPURunQueues; struct scheduler_thread_data { @@ -153,8 +154,19 @@ static int dump_run_queue(int argc, char** argv) { SimpleRunQueue::ConstIterator iterator = sRunQueue->GetConstIterator(); + kprintf("Shared run queue:\n"); dump_queue(iterator); + int32 cpuCount = smp_get_num_cpus(); + if (cpuCount < 2) + return 0; + + for (int32 i = 0; i < cpuCount; i++) { + kprintf("\nCPU %d run queue:\n", i); + sCPURunQueues[i].GetConstIterator(); + dump_queue(iterator); + } + return 0; } @@ -162,7 +174,7 @@ dump_run_queue(int argc, char** argv) static int dump_cpu_heap(int argc, char** argv) { - kprintf("\ncpu priority actual priority\n"); + kprintf("cpu priority actual priority\n"); CPUHeapEntry* entry = sCPUHeap->PeekRoot(); while (entry) { int32 cpu = entry->fCPUNumber; @@ -254,7 +266,13 @@ simple_enqueue(Thread* thread, bool newOne) T(EnqueueThread(thread, threadPriority)); - sRunQueue->PushBack(thread, threadPriority); + bool pinned = sCPURunQueues != NULL && thread->pinned_to_cpu > 0; + int32 pinnedCPU = -1; + if (pinned) { + pinnedCPU = thread->previous_cpu->cpu_num; + sCPURunQueues[pinnedCPU].PushBack(thread, threadPriority); + } else + sRunQueue->PushBack(thread, threadPriority); schedulerThreadData->cpu_bound = true; schedulerThreadData->time_left = 0; @@ -264,24 +282,32 @@ simple_enqueue(Thread* thread, bool newOne) NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - // TODO: pinned threads // TODO: disabled CPUs - CPUHeapEntry* cpuEntry = sCPUHeap->PeekRoot(); - ASSERT(cpuEntry != NULL); - int32 thisCPU = smp_get_current_cpu(); - int32 targetCPU = cpuEntry->fCPUNumber; + int32 targetCPU = pinnedCPU; + + if (!pinned) { + CPUHeapEntry* cpuEntry = sCPUHeap->PeekRoot(); + ASSERT(cpuEntry != NULL); + + targetCPU = cpuEntry->fCPUNumber; + } + + ASSERT(targetCPU >= 0); + Thread* targetThread = gCPU[targetCPU].running_thread; int32 targetPriority = simple_get_effective_priority(targetThread); ASSERT((targetCPU != thisCPU && targetThread != thread) || targetCPU == thisCPU); - int32 currentThreadPriority - = simple_get_effective_priority(thread_get_current_thread()); - if (targetPriority == currentThreadPriority) { - targetCPU = thisCPU; - targetPriority = currentThreadPriority; + if (!pinned) { + int32 currentThreadPriority + = simple_get_effective_priority(thread_get_current_thread()); + if (targetPriority == currentThreadPriority) { + targetCPU = thisCPU; + targetPriority = currentThreadPriority; + } } TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, @@ -293,7 +319,7 @@ simple_enqueue(Thread* thread, bool newOne) // It is possible that another CPU schedules the thread before the // target CPU. However, since the target CPU is sent an ICI it will // reschedule anyway and update its heap key to the correct value. - sCPUHeap->ModifyKey(cpuEntry, threadPriority); + sCPUHeap->ModifyKey(&sCPUEntries[targetCPU], threadPriority); if (targetCPU == smp_get_current_cpu()) { gCPU[targetCPU].invoke_scheduler = true; @@ -318,6 +344,21 @@ simple_enqueue_in_run_queue(Thread* thread) } +static inline void +simple_put_back(Thread* thread) +{ + bool pinned = sCPURunQueues != NULL && thread->pinned_to_cpu > 0; + + if (!pinned) + sRunQueue->PushFront(thread, simple_get_effective_priority(thread)); + else { + int32 pinnedCPU = thread->previous_cpu->cpu_num; + sCPURunQueues[pinnedCPU].PushFront(thread, + simple_get_effective_priority(thread)); + } +} + + /*! Sets the priority of a thread. Note: thread lock must be held when entering this function */ @@ -466,6 +507,36 @@ simple_compute_quantum(Thread* thread) } +static inline Thread* +simple_dequeue_thread(int32 thisCPU) +{ + Thread* sharedThread = sRunQueue->PeekMaximum(); + + Thread* pinnedThread = NULL; + if (sCPURunQueues != NULL) + pinnedThread = sCPURunQueues[thisCPU].PeekMaximum(); + + if (sharedThread == NULL && pinnedThread == NULL) + return NULL; + + int32 pinnedPriority = -1; + if (pinnedThread != NULL) + pinnedPriority = simple_get_effective_priority(pinnedThread); + + int32 sharedPriority = -1; + if (sharedThread != NULL) + sharedPriority = simple_get_effective_priority(sharedThread); + + if (sharedPriority > pinnedPriority) { + sRunQueue->Remove(sharedThread); + return sharedThread; + } + + sCPURunQueues[thisCPU].Remove(pinnedThread); + return pinnedThread; +} + + /*! Runs the scheduler. Note: expects thread spinlock to be held */ @@ -517,8 +588,7 @@ simple_reschedule(void) } else { TRACE("putting thread %ld back in run queue priority = %ld\n", oldThread->id, simple_get_effective_priority(oldThread)); - sRunQueue->PushFront(oldThread, - simple_get_effective_priority(oldThread)); + simple_put_back(oldThread); } break; @@ -539,10 +609,9 @@ simple_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - nextThread = sRunQueue->PeekMaximum(); + nextThread = simple_dequeue_thread(thisCPU); if (!nextThread) panic("reschedule(): run queues are empty!\n"); - sRunQueue->Remove(nextThread); TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, nextThread->id); @@ -676,6 +745,20 @@ scheduler_simple_init() if (result != B_OK) return result; + ArrayDeleter cpuRunQueuesDeleter; + if (cpuCount > 1) { + sCPURunQueues = new(std::nothrow) SimpleRunQueue[cpuCount]; + if (sCPURunQueues == NULL) + return B_NO_MEMORY; + cpuRunQueuesDeleter.SetTo(sCPURunQueues); + + for (int i = 0; i < cpuCount; i++) { + result = sCPURunQueues[i].GetInitStatus(); + if (result != B_OK) + return result; + } + } + gScheduler = &kSimpleOps; add_debugger_command_etc("run_queue", &dump_run_queue, @@ -687,5 +770,6 @@ scheduler_simple_init() cpuHeapDeleter.Detach(); cpuEntriesDeleter.Detach(); runQueueDeleter.Detach(); + cpuRunQueuesDeleter.Detach(); return B_OK; } From ebec24f9e0d7c5431ba1004e7aa4efcb0f093bc0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 15 Oct 2013 02:02:54 +0200 Subject: [PATCH 044/273] kernel: Add support for disabling CPUs in scheduler --- src/system/kernel/scheduler/scheduler_simple.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index fc8fe9b9b4..e1c3474c69 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -282,7 +282,6 @@ simple_enqueue(Thread* thread, bool newOne) NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - // TODO: disabled CPUs int32 thisCPU = smp_get_current_cpu(); int32 targetCPU = pinnedCPU; @@ -609,7 +608,18 @@ simple_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - nextThread = simple_dequeue_thread(thisCPU); + if (oldThread->cpu->disabled) { + ASSERT(sCPURunQueues != NULL); + nextThread = sCPURunQueues[thisCPU].PeekMaximum(); + if (nextThread != NULL) + sCPURunQueues[thisCPU].Remove(nextThread); + else { + nextThread = sRunQueue->GetHead(B_IDLE_PRIORITY); + if (nextThread != NULL) + sRunQueue->Remove(nextThread); + } + } else + nextThread = simple_dequeue_thread(thisCPU); if (!nextThread) panic("reschedule(): run queues are empty!\n"); From cf863a50401af89883ea314ccf54e16badd9439e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 16 Oct 2013 18:39:25 +0200 Subject: [PATCH 045/273] kernel: Decide whether to use simple or affine scheduler Simple scheduler is used when we do not have to worry about cache affinity (i.e. single core with or without SMT, multicore with all cache levels shared). When we replace gSchedulerLock with more fine grained locking affine scheduler should also be chosen when logical CPU count is high (regardless of cache). --- headers/private/kernel/cpu.h | 1 + src/system/kernel/arch/x86/arch_cpu.cpp | 13 ++++----- src/system/kernel/cpu.cpp | 1 + src/system/kernel/scheduler/scheduler.cpp | 35 +++++++++++++++++------ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 2432cce524..e8fe6bc490 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -76,6 +76,7 @@ typedef struct cpu_ent { //extern cpu_ent gCPU[MAX_BOOT_CPUS]; extern cpu_ent gCPU[]; +extern uint32 gCPUCacheLevelCount; #ifdef __cplusplus diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 7ea90bf672..91cc6bf31a 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -131,7 +131,6 @@ static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS]; /* Cache topology information */ static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL]; -static uint32 sCacheLevelCount; static status_t @@ -611,7 +610,7 @@ detect_amd_cache_topology(uint32 maxExtendedLeaf) for (int i = 0; i < maxCacheLevel; i++) sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); - sCacheLevelCount = maxCacheLevel; + gCPUCacheLevelCount = maxCacheLevel; } @@ -725,7 +724,7 @@ detect_intel_cache_topology(uint32 maxBasicLeaf) for (int i = 0; i < maxCacheLevel; i++) sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); - sCacheLevelCount = maxCacheLevel; + gCPUCacheLevelCount = maxCacheLevel; } @@ -789,7 +788,7 @@ detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE); unsigned int i; - for (i = 0; i < sCacheLevelCount; i++) + for (i = 0; i < gCPUCacheLevelCount; i++) cpu->cache_id[i] = topologyID & sCacheSharingMask[i]; for (; i < CPU_MAX_CACHE_LEVEL; i++) cpu->cache_id[i] = -1; @@ -800,14 +799,14 @@ detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf, cpu->topology_id[CPU_TOPOLOGY_CORE], cpu->topology_id[CPU_TOPOLOGY_SMT]); - if (sCacheLevelCount > 0) { + if (gCPUCacheLevelCount > 0) { char cacheLevels[256]; unsigned int offset = 0; - for (i = 0; i < sCacheLevelCount; i++) { + for (i = 0; i < gCPUCacheLevelCount; i++) { offset += snprintf(cacheLevels + offset, sizeof(cacheLevels) - offset, " L%d id %d%s", i + 1, cpu->cache_id[i], - i < sCacheLevelCount - 1 ? "," : ""); + i < gCPUCacheLevelCount - 1 ? "," : ""); if (offset >= sizeof(cacheLevels)) break; diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 44bff7e25d..1dfbc4dcc1 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -21,6 +21,7 @@ /* global per-cpu structure */ cpu_ent gCPU[MAX_BOOT_CPUS]; +uint32 gCPUCacheLevelCount; static spinlock sSetCpuLock; diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ba6a00825f..fd843119c3 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -59,20 +59,39 @@ scheduler_remove_listener(struct SchedulerListener* listener) } +static bool +should_use_affine_scheduler(int32 cpuCount) +{ + if (cpuCount < 2) + return false; + + for (int32 i = 1; i < cpuCount; i++) { + for (int32 j = 0; j < gCPUCacheLevelCount; j++) { + if (gCPU[i].cache_id[j] != gCPU[i - 1].cache_id[j]) + return true; + } + } + + return false; +} + + void scheduler_init(void) { int32 cpuCount = smp_get_num_cpus(); - dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s\n", cpuCount, - cpuCount != 1 ? "s" : ""); + dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s and %" B_PRId32 + " cache level%s\n", cpuCount, cpuCount != 1 ? "s" : "", + gCPUCacheLevelCount, gCPUCacheLevelCount != 1 ? "s" : ""); status_t result; -#if 0 - dprintf("scheduler_init: using affine scheduler\n"); - result = scheduler_affine_init(); -#endif - dprintf("scheduler_init: using simple scheduler\n"); - result = scheduler_simple_init(); + if (should_use_affine_scheduler(cpuCount)) { + dprintf("scheduler_init: using affine scheduler\n"); + result = scheduler_affine_init(); + } else { + dprintf("scheduler_init: using simple scheduler\n"); + result = scheduler_simple_init(); + } if (result != B_OK) panic("scheduler_init: failed to initialize scheduler\n"); From 824ed26c51252bbe23ea16252f795e011e1687fb Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 16 Oct 2013 20:02:56 +0200 Subject: [PATCH 046/273] kernel: Fully detect CPU topology before initializing scheduler --- src/system/kernel/main.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/main.cpp b/src/system/kernel/main.cpp index 6ce9648e39..938b3a61a6 100644 --- a/src/system/kernel/main.cpp +++ b/src/system/kernel/main.cpp @@ -75,6 +75,15 @@ static uint32 sCpuRendezvous3; static int32 main2(void *); +static void +non_boot_cpu_init(void* args, int currentCPU) +{ + kernel_args* kernelArgs = (kernel_args*)args; + if (currentCPU != 0) + cpu_init_percpu(kernelArgs, currentCPU); +} + + extern "C" int _start(kernel_args *bootKernelArgs, int currentCPU) { @@ -149,6 +158,8 @@ _start(kernel_args *bootKernelArgs, int currentCPU) int_init_post_vm(&sKernelArgs); cpu_init_post_vm(&sKernelArgs); commpage_init(); + call_all_cpus_sync(non_boot_cpu_init, &sKernelArgs); + TRACE("init system info\n"); system_info_init(&sKernelArgs); @@ -229,7 +240,6 @@ _start(kernel_args *bootKernelArgs, int currentCPU) arch_cpu_global_TLB_invalidate(); // this is run for each non boot processor after they've been set loose - cpu_init_percpu(&sKernelArgs, currentCPU); smp_per_cpu_init(&sKernelArgs, currentCPU); // wait for all other AP cpus to get to this point From 3ec1d8da429b162235dc0f5e4023cc71cdc0533f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 16 Oct 2013 23:44:35 +0200 Subject: [PATCH 047/273] scheduler_affine: Add logic shared with simple scheduler The scheduler is in very early stage. There is no thread migration and the algorithms choosing CPU for thread are very simple. Since affine scheduler is going to use one run queue per core simple on single core machines it will work exactly the same as simple scheduler. That would allow us to have only one scheduler implementation usable on all kinds of machines. --- .../kernel/scheduler/scheduler_affine.cpp | 812 +++++++++++++----- .../kernel/scheduler/scheduler_simple.cpp | 9 +- 2 files changed, 606 insertions(+), 215 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 471b53ef7c..d370847a30 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013. Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2009, Rene Gollent, rene@gollent.com. * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. @@ -15,6 +16,7 @@ #include +#include #include #include #include @@ -25,98 +27,258 @@ #include #include #include +#include #include +#include "RunQueue.h" #include "scheduler_common.h" #include "scheduler_tracing.h" //#define TRACE_SCHEDULER #ifdef TRACE_SCHEDULER -# define TRACE(x) dprintf_no_syslog x +# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) #else -# define TRACE(x) ; +# define TRACE(...) do { } while (false) #endif -// The run queues. Holds the threads ready to run ordered by priority. -// One queue per schedulable target (CPU, core, etc.). -// TODO: consolidate this such that HT/SMT entities on the same physical core -// share a queue, once we have the necessary API for retrieving the topology -// information -static Thread* sRunQueue[B_MAX_CPU_COUNT]; -static int32 sRunQueueSize[B_MAX_CPU_COUNT]; -static Thread* sIdleThreads; -const int32 kMaxTrackingQuantums = 5; +const bigtime_t kThreadQuantum = 1000; const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; -struct scheduler_thread_data { - scheduler_thread_data(void) - { - Init(); - } - - - void Init() - { - fQuantumAverage = 0; - fLastQuantumSlot = 0; - fLastQueue = -1; - memset(fLastThreadQuantums, 0, sizeof(fLastThreadQuantums)); - } - - inline void SetQuantum(int32 quantum) - { - fQuantumAverage -= fLastThreadQuantums[fLastQuantumSlot]; - fLastThreadQuantums[fLastQuantumSlot] = quantum; - fQuantumAverage += quantum; - if (fLastQuantumSlot < kMaxTrackingQuantums - 1) - ++fLastQuantumSlot; - else - fLastQuantumSlot = 0; - } - - inline int32 GetAverageQuantumUsage() const - { - return fQuantumAverage / kMaxTrackingQuantums; - } - - int32 fQuantumAverage; - int32 fLastThreadQuantums[kMaxTrackingQuantums]; - int16 fLastQuantumSlot; - int32 fLastQueue; +struct CPUHeapEntry : public HeapLinkImpl { + int32 fCPUNumber; }; +static CPUHeapEntry* sCPUEntries; +typedef Heap AffineCPUHeap; +static AffineCPUHeap* sCPUHeap; + +// The run queues. Holds the threads ready to run ordered by priority. +// One queue per schedulable target per core. Additionally, each +// logical processor has its sCPURunQueues used for scheduling +// pinned threads. +typedef RunQueue AffineRunQueue; +static AffineRunQueue* sRunQueues; +static AffineRunQueue* sCPURunQueues; +static int32 sRunQueueCount; +static int32* sCPUToCore; + + +struct scheduler_thread_data { + scheduler_thread_data() { Init(); } + inline void Init(); + + int32 priority_penalty; + int32 additional_penalty; + + bool lost_cpu; + bool cpu_bound; + + bigtime_t time_left; + bigtime_t stolen_time; + bigtime_t quantum_start; + + bigtime_t went_sleep; + + int32 previous_core; +}; + + +void +scheduler_thread_data::Init() +{ + priority_penalty = 0; + additional_penalty = 0; + + time_left = 0; + stolen_time = 0; + + went_sleep = 0; + + lost_cpu = false; + cpu_bound = true; + + previous_core = -1; +} + + +static inline int +affine_get_minimal_priority(Thread* thread) +{ + return min_c(thread->priority, 25) / 5; +} + + +static inline int32 +affine_get_thread_penalty(Thread* thread) +{ + int32 penalty = thread->scheduler_data->priority_penalty; + + const int kMinimalPriority = affine_get_minimal_priority(thread); + if (kMinimalPriority > 0) { + penalty + += thread->scheduler_data->additional_penalty % kMinimalPriority; + } + + return penalty; + +} + + +static inline int32 +affine_get_effective_priority(Thread* thread) +{ + if (thread->priority == B_IDLE_PRIORITY) + return thread->priority; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return thread->priority; + + int32 effectivePriority = thread->priority; + effectivePriority -= affine_get_thread_penalty(thread); + + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return effectivePriority; +} + + +static void +dump_queue(AffineRunQueue::ConstIterator& iterator) +{ + if (!iterator.HasNext()) + kprintf("Run queue is empty.\n"); + else { + kprintf("thread id priority penalty name\n"); + while (iterator.HasNext()) { + Thread* thread = iterator.Next(); + kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", + thread, thread->id, thread->priority, + affine_get_thread_penalty(thread), thread->name); + } + } +} + static int dump_run_queue(int argc, char **argv) { - Thread *thread = NULL; + int32 cpuCount = smp_get_num_cpus(); + int32 coreCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) + sCPUToCore[i] = coreCount++; + } - for (int32 i = 0; i < smp_get_num_cpus(); i++) { - thread = sRunQueue[i]; - kprintf("Run queue for cpu %" B_PRId32 " (%" B_PRId32 " threads)\n", i, - sRunQueueSize[i]); - if (sRunQueueSize[i] > 0) { - kprintf("thread id priority avg. quantum name\n"); - while (thread) { - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-12" B_PRId32 - " %s\n", thread, thread->id, thread->priority, - thread->scheduler_data->GetAverageQuantumUsage(), - thread->name); - thread = thread->queue_next; - } + AffineRunQueue::ConstIterator iterator; + for (int32 i = 0; i < coreCount; i++) { + kprintf("\nCore %" B_PRId32 " run queue:\n", i); + iterator = sRunQueues[i].GetConstIterator(); + dump_queue(iterator); + } + + for (int32 i = 0; i < cpuCount; i++) { + iterator = sCPURunQueues[i].GetConstIterator(); + + if (iterator.HasNext()) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", i); + dump_queue(iterator); } } + return 0; } +static int +dump_cpu_heap(int argc, char** argv) +{ + kprintf("cpu priority actual priority\n"); + CPUHeapEntry* entry = sCPUHeap->PeekRoot(); + while (entry) { + int32 cpu = entry->fCPUNumber; + kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, + sCPUHeap->GetKey(entry), + affine_get_effective_priority(gCPU[cpu].running_thread)); + + sCPUHeap->RemoveRoot(); + entry = sCPUHeap->PeekRoot(); + } + + int32 cpuCount = smp_get_num_cpus(); + for (int i = 0; i < cpuCount; i++) { + sCPUHeap->Insert(&sCPUEntries[i], + affine_get_effective_priority(gCPU[i].running_thread)); + } + + return 0; +} + + +static void +affine_dump_thread_data(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", + schedulerThreadData->priority_penalty); + + int32 additionalPenalty = 0; + const int kMinimalPriority = affine_get_minimal_priority(thread); + if (kMinimalPriority > 0) { + additionalPenalty + = schedulerThreadData->additional_penalty % kMinimalPriority; + } + kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", + additionalPenalty, schedulerThreadData->additional_penalty); + kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", + schedulerThreadData->stolen_time); + kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", + schedulerThreadData->previous_core); +} + + +static inline void +affine_increase_penalty(Thread* thread) +{ + if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) + return; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return; + + TRACE("increasing thread %ld penalty\n", thread->id); + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + int32 oldPenalty = schedulerThreadData->priority_penalty++; + + ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); + const int kMinimalPriority = affine_get_minimal_priority(thread); + if (thread->priority - oldPenalty <= kMinimalPriority) { + schedulerThreadData->priority_penalty = oldPenalty; + schedulerThreadData->additional_penalty++; + } +} + + +static inline void +affine_cancel_penalty(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + if (schedulerThreadData->priority_penalty != 0) + TRACE("cancelling thread %ld penalty\n", thread->id); + schedulerThreadData->priority_penalty = 0; + schedulerThreadData->additional_penalty = 0; +} + + /*! Returns the most idle CPU based on the active time counters. Note: thread lock must be held when entering this function */ +#if 0 static int32 affine_get_most_idle_cpu() { @@ -130,54 +292,79 @@ affine_get_most_idle_cpu() return targetCPU; } +#endif -/*! Enqueues the thread into the run queue. - Note: thread lock must be held when entering this function -*/ static void -affine_enqueue_in_run_queue(Thread *thread) +affine_enqueue(Thread* thread, bool newOne) { - int32 targetCPU = -1; - if (thread->pinned_to_cpu > 0) - targetCPU = thread->previous_cpu->cpu_num; - else if (thread->previous_cpu == NULL || thread->previous_cpu->disabled) - targetCPU = affine_get_most_idle_cpu(); - else - targetCPU = thread->previous_cpu->cpu_num; - thread->state = thread->next_state = B_THREAD_READY; - if (thread->priority == B_IDLE_PRIORITY) { - thread->queue_next = sIdleThreads; - sIdleThreads = thread; - } else { - Thread *curr, *prev; - for (curr = sRunQueue[targetCPU], prev = NULL; curr - && curr->priority >= thread->priority; - curr = curr->queue_next) { - if (prev) - prev = prev->queue_next; - else - prev = sRunQueue[targetCPU]; + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; + if (newOne && hasSlept > kThreadQuantum) + affine_cancel_penalty(thread); + + int32 threadPriority = affine_get_effective_priority(thread); + + T(EnqueueThread(thread, threadPriority)); + + bool pinned = thread->pinned_to_cpu > 0; + int32 targetCPU = -1; + int32 targetCore; + if (pinned) { + targetCPU = thread->previous_cpu->cpu_num; + targetCore = sCPUToCore[targetCPU]; + ASSERT(targetCore == schedulerThreadData->previous_core); + } else if (schedulerThreadData->previous_core < 0) { + if (thread->priority == B_IDLE_PRIORITY) { + static int32 idleThreads = 0; + targetCPU = idleThreads++; + targetCore = sCPUToCore[targetCPU]; + } else { + CPUHeapEntry* cpuEntry = sCPUHeap->PeekRoot(); + ASSERT(cpuEntry != NULL); + + targetCPU = cpuEntry->fCPUNumber; + targetCore = sCPUToCore[targetCPU]; } - - T(EnqueueThread(thread, thread->priority)); - sRunQueueSize[targetCPU]++; - thread->queue_next = curr; - if (prev) - prev->queue_next = thread; - else - sRunQueue[targetCPU] = thread; - - thread->scheduler_data->fLastQueue = targetCPU; + schedulerThreadData->previous_core = targetCore; + } else { + targetCPU = thread->previous_cpu->cpu_num; + targetCore = sCPUToCore[targetCPU]; + ASSERT(targetCore == schedulerThreadData->previous_core); } + TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, + threadPriority, targetCore); + if (pinned) + sCPURunQueues[targetCPU].PushBack(thread, threadPriority); + else + sRunQueues[targetCore].PushBack(thread, threadPriority); + + schedulerThreadData->cpu_bound = true; + schedulerThreadData->time_left = 0; + schedulerThreadData->stolen_time = 0; + // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - if (thread->priority > gCPU[targetCPU].running_thread->priority) { + Thread* targetThread = gCPU[targetCPU].running_thread; + int32 targetPriority = affine_get_effective_priority(targetThread); + + TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, + targetPriority); + + if (threadPriority > targetPriority) { + targetThread->scheduler_data->lost_cpu = true; + + // It is possible that another CPU schedules the thread before the + // target CPU. However, since the target CPU is sent an ICI it will + // reschedule anyway and update its heap key to the correct value. + sCPUHeap->ModifyKey(&sCPUEntries[targetCPU], threadPriority); + if (targetCPU == smp_get_current_cpu()) { gCPU[targetCPU].invoke_scheduler = true; gCPU[targetCPU].invoke_scheduler_if_idle = false; @@ -189,6 +376,37 @@ affine_enqueue_in_run_queue(Thread *thread) } +/*! Enqueues the thread into the run queue. + Note: thread lock must be held when entering this function +*/ +static void +affine_enqueue_in_run_queue(Thread *thread) +{ + TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, + thread->priority); + affine_enqueue(thread, true); +} + + +static inline void +affine_put_back(Thread* thread) +{ + bool pinned = sCPURunQueues != NULL && thread->pinned_to_cpu > 0; + + if (pinned) { + int32 pinnedCPU = thread->previous_cpu->cpu_num; + sCPURunQueues[pinnedCPU].PushFront(thread, + affine_get_effective_priority(thread)); + } else { + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); + sRunQueues[previousCore].PushFront(thread, + affine_get_effective_priority(thread)); + } +} + + +#if 0 /*! Dequeues the thread after the given \a prevThread from the run queue. */ static inline Thread * @@ -257,6 +475,7 @@ steal_thread_from_other_cpus(int32 currentCPU) return NULL; } +#endif /*! Sets the priority of a thread. @@ -265,12 +484,18 @@ steal_thread_from_other_cpus(int32 currentCPU) static void affine_set_thread_priority(Thread *thread, int32 priority) { - int32 targetCPU = -1; - if (priority == thread->priority) return; + TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", + thread->id, priority, thread->priority, + affine_get_effective_priority(thread)); + + if (thread->state == B_THREAD_RUNNING) + sCPUHeap->ModifyKey(&sCPUEntries[thread->cpu->cpu_num], priority); + if (thread->state != B_THREAD_READY) { + affine_cancel_penalty(thread); thread->priority = priority; return; } @@ -284,25 +509,15 @@ affine_set_thread_priority(Thread *thread, int32 priority) NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, thread); - // search run queues for the thread - Thread *item = NULL, *prev = NULL; - targetCPU = thread->scheduler_data->fLastQueue; - - for (item = sRunQueue[targetCPU], prev = NULL; item && item != thread; - item = item->queue_next) { - if (prev) - prev = prev->queue_next; - else - prev = item; - } - - ASSERT(item == thread); - - // remove the thread - thread = dequeue_from_run_queue(prev, targetCPU); + // remove thread from run queue + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); + sRunQueues[previousCore].Remove(thread); // set priority and re-insert + affine_cancel_penalty(thread); thread->priority = priority; + affine_enqueue_in_run_queue(thread); } @@ -330,21 +545,129 @@ reschedule_event(timer *unused) { // This function is called as a result of the timer event set by the // scheduler. Make sure the reschedule() is invoked. - thread_get_current_thread()->cpu->invoke_scheduler = true; - thread_get_current_thread()->cpu->invoke_scheduler_if_idle = false; - thread_get_current_thread()->cpu->preempted = 1; + Thread* thread= thread_get_current_thread(); + + thread->scheduler_data->lost_cpu = true; + thread->cpu->invoke_scheduler = true; + thread->cpu->invoke_scheduler_if_idle = false; + thread->cpu->preempted = 1; return B_HANDLED_INTERRUPT; } +static inline bool +affine_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + if (hasYielded) { + schedulerThreadData->time_left = 0; + return true; + } + + bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; + schedulerThreadData->time_left -= time_used; + schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); + + // too little time left, it's better make the next quantum a bit longer + if (wasPreempted || schedulerThreadData->time_left <= kThreadQuantum / 50) { + schedulerThreadData->stolen_time += schedulerThreadData->time_left; + schedulerThreadData->time_left = 0; + } + + return schedulerThreadData->time_left == 0; +} + + +static inline bigtime_t +affine_quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, + int32 maxPriority, int32 minPriority, int32 priority) +{ + ASSERT(priority <= maxPriority); + ASSERT(priority >= minPriority); + + bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); + result /= maxPriority - minPriority; + return maxQuantum - result; +} + + +static inline bigtime_t +affine_get_base_quantum(Thread* thread) +{ + int32 priority = affine_get_effective_priority(thread); + + if (priority >= B_URGENT_DISPLAY_PRIORITY) + return kThreadQuantum; + if (priority > B_NORMAL_PRIORITY) { + return affine_quantum_linear_interpolation(kThreadQuantum * 4, + kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, + priority); + } + return affine_quantum_linear_interpolation(kThreadQuantum * 64, + kThreadQuantum * 4, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); +} + + +static inline bigtime_t +affine_compute_quantum(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + bigtime_t quantum; + if (schedulerThreadData->time_left != 0) + quantum = schedulerThreadData->time_left; + else + quantum = affine_get_base_quantum(thread); + + quantum += schedulerThreadData->stolen_time; + schedulerThreadData->stolen_time = 0; + + schedulerThreadData->time_left = quantum; + schedulerThreadData->quantum_start = system_time(); + + return quantum; +} + + +static inline Thread* +affine_dequeue_thread(int32 thisCPU) +{ + int32 thisCore = sCPUToCore[thisCPU]; + Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); + + Thread* pinnedThread = NULL; + if (sCPURunQueues != NULL) + pinnedThread = sCPURunQueues[thisCPU].PeekMaximum(); + + if (sharedThread == NULL && pinnedThread == NULL) + return NULL; + + int32 pinnedPriority = -1; + if (pinnedThread != NULL) + pinnedPriority = affine_get_effective_priority(pinnedThread); + + int32 sharedPriority = -1; + if (sharedThread != NULL) + sharedPriority = affine_get_effective_priority(sharedThread); + + if (sharedPriority > pinnedPriority) { + sRunQueues[thisCore].Remove(sharedThread); + return sharedThread; + } + + sCPURunQueues[thisCPU].Remove(pinnedThread); + return pinnedThread; +} + + /*! Runs the scheduler. Note: expects thread spinlock to be held */ static void affine_reschedule(void) { - int32 currentCPU = smp_get_current_cpu(); - Thread *oldThread = thread_get_current_thread(); + Thread* oldThread = thread_get_current_thread(); // check whether we're only supposed to reschedule, if the current thread // is idle @@ -357,87 +680,77 @@ affine_reschedule(void) } } - Thread *nextThread, *prevThread; + int32 thisCPU = smp_get_current_cpu(); + int32 thisCore = sCPUToCore[thisCPU]; - TRACE(("reschedule(): cpu %ld, cur_thread = %ld\n", currentCPU, oldThread->id)); + TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, + oldThread->id); oldThread->state = oldThread->next_state; + scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; + + // update CPU heap so that old thread would have CPU properly chosen + Thread* nextThread = sRunQueues[thisCore].PeekMaximum(); + if (nextThread != NULL) { + sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], + affine_get_effective_priority(nextThread)); + } + switch (oldThread->next_state) { case B_THREAD_RUNNING: case B_THREAD_READY: - TRACE(("enqueueing thread %ld into run q. pri = %ld\n", oldThread->id, oldThread->priority)); - affine_enqueue_in_run_queue(oldThread); + if (!schedulerOldThreadData->lost_cpu) + schedulerOldThreadData->cpu_bound = false; + + if (affine_quantum_ended(oldThread, oldThread->cpu->preempted, + oldThread->has_yielded)) { + if (schedulerOldThreadData->cpu_bound) + affine_increase_penalty(oldThread); + + TRACE("enqueueing thread %ld into run queue priority = %ld\n", + oldThread->id, affine_get_effective_priority(oldThread)); + affine_enqueue(oldThread, false); + } else { + TRACE("putting thread %ld back in run queue priority = %ld\n", + oldThread->id, affine_get_effective_priority(oldThread)); + affine_put_back(oldThread); + } + break; case B_THREAD_SUSPENDED: - TRACE(("reschedule(): suspending thread %ld\n", oldThread->id)); + schedulerOldThreadData->went_sleep = system_time(); + TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: break; default: - TRACE(("not enqueueing thread %ld into run q. next_state = %ld\n", oldThread->id, oldThread->next_state)); + schedulerOldThreadData->went_sleep = system_time(); + TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", + oldThread->id, oldThread->next_state); break; } - nextThread = sRunQueue[currentCPU]; - prevThread = NULL; + oldThread->has_yielded = false; + schedulerOldThreadData->lost_cpu = false; - if (sRunQueue[currentCPU] != NULL) { - TRACE(("dequeueing next thread from cpu %ld\n", currentCPU)); - // select next thread from the run queue - while (nextThread->queue_next) { - // always extract real time threads - if (nextThread->priority >= B_FIRST_REAL_TIME_PRIORITY) - break; - - // find next thread with lower priority - Thread *lowerNextThread = nextThread->queue_next; - Thread *lowerPrevThread = nextThread; - int32 priority = nextThread->priority; - - while (lowerNextThread != NULL - && priority == lowerNextThread->priority) { - lowerPrevThread = lowerNextThread; - lowerNextThread = lowerNextThread->queue_next; - } - if (lowerNextThread == NULL) - break; - - int32 priorityDiff = priority - lowerNextThread->priority; - if (priorityDiff > 15) - break; - - // skip normal threads sometimes - // (twice as probable per priority level) - if ((fast_random_value() >> (15 - priorityDiff)) != 0) - break; - - nextThread = lowerNextThread; - prevThread = lowerPrevThread; + // select thread with the biggest priority + if (oldThread->cpu->disabled) { + ASSERT(sCPURunQueues != NULL); + nextThread = sCPURunQueues[thisCPU].PeekMaximum(); + if (nextThread != NULL) + sCPURunQueues[thisCPU].Remove(nextThread); + else { + nextThread = sRunQueues[thisCore].GetHead(B_IDLE_PRIORITY); + if (nextThread != NULL) + sRunQueues[thisCore].Remove(nextThread); } - - TRACE(("dequeuing thread %ld from cpu %ld\n", nextThread->id, - currentCPU)); - // extract selected thread from the run queue - dequeue_from_run_queue(prevThread, currentCPU); - } else { - if (!gCPU[currentCPU].disabled) { - TRACE(("CPU %ld stealing thread from other CPUs\n", currentCPU)); - nextThread = steal_thread_from_other_cpus(currentCPU); - } else - nextThread = NULL; - - if (nextThread == NULL) { - TRACE(("No threads to steal, grabbing from idle pool\n")); - // no other CPU had anything for us to take, - // grab one from the kernel's idle pool - nextThread = sIdleThreads; - if (nextThread) - sIdleThreads = nextThread->queue_next; - } - } - + } else + nextThread = affine_dequeue_thread(thisCPU); if (!nextThread) - panic("reschedule(): run queue is empty!\n"); + panic("reschedule(): run queues are empty!\n"); + + TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, + nextThread->id); T(ScheduleThread(nextThread, oldThread)); @@ -445,21 +758,23 @@ affine_reschedule(void) NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, oldThread, nextThread); + // update CPU heap + sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], + affine_get_effective_priority(nextThread)); + nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; - oldThread->has_yielded = false; + ASSERT(nextThread->scheduler_data->previous_core == thisCore); + //nextThread->scheduler_data->previous_core = thisCore; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); // track CPU activity if (!thread_is_idle_thread(oldThread)) { - bigtime_t activeTime = + oldThread->cpu->active_time += (oldThread->kernel_time - oldThread->cpu->last_kernel_time) + (oldThread->user_time - oldThread->cpu->last_user_time); - oldThread->cpu->active_time += activeTime; - scheduler_thread_data *data = oldThread->scheduler_data; - data->SetQuantum(activeTime); } if (!thread_is_idle_thread(nextThread)) { @@ -468,25 +783,13 @@ affine_reschedule(void) } if (nextThread != oldThread || oldThread->cpu->preempted) { - timer *quantumTimer = &oldThread->cpu->quantum_timer; + timer* quantumTimer = &oldThread->cpu->quantum_timer; if (!oldThread->cpu->preempted) cancel_timer(quantumTimer); + oldThread->cpu->preempted = 0; - - // we do not adjust the quantum for the idle thread as it is going to be - // preempted most of the time and would likely get the longer quantum - // over time, indeed we use a smaller quantum to avoid running idle too - // long - bigtime_t quantum = kMinThreadQuantum; - // give CPU-bound background threads a larger quantum size - // to minimize unnecessary context switches if the system is idle - if (nextThread->priority != B_IDLE_PRIORITY - && nextThread->scheduler_data->GetAverageQuantumUsage() - > (kMinThreadQuantum >> 1) - && nextThread->priority < B_NORMAL_PRIORITY) - quantum = kMaxThreadQuantum; - if (!thread_is_idle_thread(nextThread)) { + bigtime_t quantum = affine_compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); } @@ -500,13 +803,7 @@ affine_reschedule(void) static status_t affine_on_thread_create(Thread* thread, bool idleThread) { - // we don't need a data structure for the idle threads - if (idleThread) { - thread->scheduler_data = NULL; - return B_OK; - } - - thread->scheduler_data = new(std::nothrow) scheduler_thread_data(); + thread->scheduler_data = new (std::nothrow)scheduler_thread_data; if (thread->scheduler_data == NULL) return B_NO_MEMORY; return B_OK; @@ -516,7 +813,7 @@ affine_on_thread_create(Thread* thread, bool idleThread) static void affine_on_thread_init(Thread* thread) { - ((scheduler_thread_data *)(thread->scheduler_data))->Init(); + thread->scheduler_data->Init(); } @@ -548,7 +845,7 @@ static scheduler_ops kAffineOps = { affine_on_thread_init, affine_on_thread_destroy, affine_start, - NULL + affine_dump_thread_data }; @@ -558,11 +855,102 @@ static scheduler_ops kAffineOps = { status_t scheduler_affine_init() { + int32 cpuCount = smp_get_num_cpus(); + + sCPUHeap = new AffineCPUHeap; + if (sCPUHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter cpuHeapDeleter(sCPUHeap); + + sCPUEntries = new CPUHeapEntry[cpuCount]; + if (sCPUEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuEntriesDeleter(sCPUEntries); + + for (int i = 0; i < cpuCount; i++) { + sCPUEntries[i].fCPUNumber = i; + status_t result = sCPUHeap->Insert(&sCPUEntries[i], B_IDLE_PRIORITY); + if (result != B_OK) + return result; + } + + TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-cpu queue%s\n", + cpuCount, cpuCount != 1 ? "s" : ""); + + sCPUToCore = new(std::nothrow) int32[cpuCount]; + if (sCPUToCore == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuToCoreDeleter(sCPUToCore); + + sCPURunQueues = new(std::nothrow) AffineRunQueue[cpuCount]; + if (sCPURunQueues == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuRunQueuesDeleter(sCPURunQueues); + for (int i = 0; i < cpuCount; i++) { + status_t result = sCPURunQueues[i].GetInitStatus(); + if (result != B_OK) + return result; + } + + int32 coreCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) + sCPUToCore[i] = coreCount++; + } + sRunQueueCount = coreCount; + + // TODO: Nasty O(n^2), solutions with better complexity will require + // creating helper data structures. This code is run only once, so it + // probably won't be a problem until we support systems with large + // number of processors. + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) + continue; + + for (int32 j = 0; j < cpuCount; j++) { + bool sameCore = true; + for (int32 k = 0; k < CPU_TOPOLOGY_LEVELS && sameCore; k++) { + if (k == CPU_TOPOLOGY_SMT && gCPU[j].topology_id[k] == 0) + continue; + if (k == CPU_TOPOLOGY_SMT && gCPU[j].topology_id[k] != 0) { + sameCore = false; + continue; + } + + if (gCPU[i].topology_id[k] != gCPU[j].topology_id[k]) + sameCore = false; + } + + if (sameCore) + sCPUToCore[i] = sCPUToCore[j]; + } + } + + TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-core queue%s\n", + coreCount, coreCount != 1 ? "s" : ""); + + sRunQueues = new(std::nothrow) AffineRunQueue[coreCount]; + if (sRunQueues == NULL) + return B_NO_MEMORY; + ArrayDeleter runQueuesDeleter(sRunQueues); + for (int i = 0; i < coreCount; i++) { + status_t result = sRunQueues[i].GetInitStatus(); + if (result != B_OK) + return result; + } + gScheduler = &kAffineOps; - memset(sRunQueue, 0, sizeof(sRunQueue)); - memset(sRunQueueSize, 0, sizeof(sRunQueueSize)); + add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", + 0); + cpuHeapDeleter.Detach(); + cpuEntriesDeleter.Detach(); + cpuToCoreDeleter.Detach(); + cpuRunQueuesDeleter.Detach(); + runQueuesDeleter.Detach(); return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index e1c3474c69..951994e2dd 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -162,9 +162,12 @@ dump_run_queue(int argc, char** argv) return 0; for (int32 i = 0; i < cpuCount; i++) { - kprintf("\nCPU %d run queue:\n", i); - sCPURunQueues[i].GetConstIterator(); - dump_queue(iterator); + iterator = sCPURunQueues[i].GetConstIterator(); + + if (iterator.HasNext()) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", i); + dump_queue(iterator); + } } return 0; From 278c9784a13e80068e7ab97ffd9feb27b98e17d6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 17 Oct 2013 01:50:14 +0200 Subject: [PATCH 048/273] scheduler_affine: Use global core heap and per-core CPU heaps There is a global heap of cores, where the key is the highest priority of threads running on that core. Moreover, for each core there is a heap of logical processors on this core where the key is the priority of currently running thread. The per-core heap is used for load balancing among logical processors on that core. The global heap is used in initial decision where to put the thread (note that the algorithm that makes this decision is not complete yet). --- headers/private/kernel/util/Heap.h | 14 +- .../kernel/scheduler/scheduler_affine.cpp | 272 +++++++++++++----- 2 files changed, 204 insertions(+), 82 deletions(-) diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h index cca75c7569..653e4b09e8 100644 --- a/headers/private/kernel/util/Heap.h +++ b/headers/private/kernel/util/Heap.h @@ -79,7 +79,7 @@ public: inline Element* PeekRoot(); - inline const Key& GetKey(Element* element) const; + static const Key& GetKey(Element* element); inline void ModifyKey(Element* element, Key newKey); @@ -96,9 +96,8 @@ private: int fLastElement; int fSize; - Compare sCompare; - GetLink sGetLink; - + static Compare sCompare; + static GetLink sGetLink; }; @@ -187,12 +186,9 @@ HEAP_CLASS_NAME::PeekRoot() HEAP_TEMPLATE_LIST const Key& -HEAP_CLASS_NAME::GetKey(Element* element) const +HEAP_CLASS_NAME::GetKey(Element* element) { - HeapLink* link = sGetLink(element); - - ASSERT(link->fIndex >= 0 && link->fIndex < fLastElement); - return link->fKey; + return sGetLink(element)->fKey; } diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index d370847a30..b1667bf07b 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -49,20 +49,35 @@ const bigtime_t kMaxThreadQuantum = 10000; struct CPUHeapEntry : public HeapLinkImpl { + HeapLink fMaxHeap; int32 fCPUNumber; }; -static CPUHeapEntry* sCPUEntries; +static CPUHeapEntry* sCPUPriorityEntries; typedef Heap AffineCPUHeap; -static AffineCPUHeap* sCPUHeap; +typedef Heap, + HeapMemberGetLink > + AffineCPUMaxHeap; + +// TODO: Use one min-max heap per-core +static AffineCPUHeap* sCPUPriorityHeaps; +static AffineCPUMaxHeap* sCPUMaxPriorityHeaps; + +struct CoreHeapEntry : public HeapLinkImpl { + int32 fCoreID; +}; + +static CoreHeapEntry* sCorePriorityEntries; +typedef Heap AffineCoreHeap; +static AffineCoreHeap* sCorePriorityHeap; // The run queues. Holds the threads ready to run ordered by priority. // One queue per schedulable target per core. Additionally, each -// logical processor has its sCPURunQueues used for scheduling +// logical processor has its sPinnedRunQueues used for scheduling // pinned threads. typedef RunQueue AffineRunQueue; static AffineRunQueue* sRunQueues; -static AffineRunQueue* sCPURunQueues; +static AffineRunQueue* sPinnedRunQueues; static int32 sRunQueueCount; static int32* sCPUToCore; @@ -181,7 +196,7 @@ dump_run_queue(int argc, char **argv) } for (int32 i = 0; i < cpuCount; i++) { - iterator = sCPURunQueues[i].GetConstIterator(); + iterator = sPinnedRunQueues[i].GetConstIterator(); if (iterator.HasNext()) { kprintf("\nCPU %" B_PRId32 " run queue:\n", i); @@ -193,25 +208,64 @@ dump_run_queue(int argc, char **argv) } +static void +dump_heap(AffineCPUHeap* heap) +{ + AffineCPUHeap temp; + + kprintf("cpu priority actual priority\n"); + CPUHeapEntry* entry = heap->PeekRoot(); + while (entry) { + int32 cpu = entry->fCPUNumber; + int32 key = AffineCPUHeap::GetKey(entry); + kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, key, + affine_get_effective_priority(gCPU[cpu].running_thread)); + + heap->RemoveRoot(); + temp.Insert(entry, key); + + entry = heap->PeekRoot(); + } + + entry = temp.PeekRoot(); + while (entry) { + int32 key = AffineCPUHeap::GetKey(entry); + temp.RemoveRoot(); + heap->Insert(entry, key); + entry = temp.PeekRoot(); + } +} + + static int dump_cpu_heap(int argc, char** argv) { - kprintf("cpu priority actual priority\n"); - CPUHeapEntry* entry = sCPUHeap->PeekRoot(); - while (entry) { - int32 cpu = entry->fCPUNumber; - kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, - sCPUHeap->GetKey(entry), - affine_get_effective_priority(gCPU[cpu].running_thread)); + AffineCoreHeap temp; - sCPUHeap->RemoveRoot(); - entry = sCPUHeap->PeekRoot(); + kprintf("core priority\n"); + CoreHeapEntry* entry = sCorePriorityHeap->PeekRoot(); + while (entry) { + int32 core = entry->fCoreID; + int32 key = AffineCoreHeap::GetKey(entry); + kprintf("%4" B_PRId32 " %8" B_PRId32 "\n", core, key); + + sCorePriorityHeap->RemoveRoot(); + temp.Insert(entry, key); + + entry = sCorePriorityHeap->PeekRoot(); } - int32 cpuCount = smp_get_num_cpus(); - for (int i = 0; i < cpuCount; i++) { - sCPUHeap->Insert(&sCPUEntries[i], - affine_get_effective_priority(gCPU[i].running_thread)); + entry = temp.PeekRoot(); + while (entry) { + int32 key = AffineCoreHeap::GetKey(entry); + temp.RemoveRoot(); + sCorePriorityHeap->Insert(entry, key); + entry = temp.PeekRoot(); + } + + for (int32 i = 0; i < sRunQueueCount; i++) { + kprintf("\nCore %" B_PRId32 " heap:\n", i); + dump_heap(&sCPUPriorityHeaps[i]); } return 0; @@ -295,6 +349,41 @@ affine_get_most_idle_cpu() #endif +static inline void +affine_update_priority_heaps(int32 cpu, int32 priority) +{ + int32 core = sCPUToCore[cpu]; + + sCPUPriorityHeaps[core].ModifyKey(&sCPUPriorityEntries[cpu], priority); + sCPUMaxPriorityHeaps[core].ModifyKey(&sCPUPriorityEntries[cpu], priority); + + int32 maxPriority + = AffineCPUMaxHeap::GetKey(sCPUMaxPriorityHeaps[core].PeekRoot()); + int32 corePriority = AffineCoreHeap::GetKey(&sCorePriorityEntries[core]); + + if (corePriority != maxPriority) + sCorePriorityHeap->ModifyKey(&sCorePriorityEntries[core], maxPriority); +} + + +static inline int32 +affine_choose_core(void) +{ + CoreHeapEntry* entry = sCorePriorityHeap->PeekRoot(); + ASSERT(entry != NULL); + return entry->fCoreID; +} + + +static inline int32 +affine_choose_cpu(int32 core) +{ + CPUHeapEntry* entry = sCPUPriorityHeaps[core].PeekRoot(); + ASSERT(entry != NULL); + return entry->fCPUNumber; +} + + static void affine_enqueue(Thread* thread, bool newOne) { @@ -323,23 +412,19 @@ affine_enqueue(Thread* thread, bool newOne) targetCPU = idleThreads++; targetCore = sCPUToCore[targetCPU]; } else { - CPUHeapEntry* cpuEntry = sCPUHeap->PeekRoot(); - ASSERT(cpuEntry != NULL); - - targetCPU = cpuEntry->fCPUNumber; - targetCore = sCPUToCore[targetCPU]; + targetCore = affine_choose_core(); + targetCPU = affine_choose_cpu(targetCore); } schedulerThreadData->previous_core = targetCore; } else { - targetCPU = thread->previous_cpu->cpu_num; - targetCore = sCPUToCore[targetCPU]; - ASSERT(targetCore == schedulerThreadData->previous_core); + targetCore = schedulerThreadData->previous_core; + targetCPU = affine_choose_cpu(targetCore); } TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, threadPriority, targetCore); if (pinned) - sCPURunQueues[targetCPU].PushBack(thread, threadPriority); + sPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); else sRunQueues[targetCore].PushBack(thread, threadPriority); @@ -363,7 +448,7 @@ affine_enqueue(Thread* thread, bool newOne) // It is possible that another CPU schedules the thread before the // target CPU. However, since the target CPU is sent an ICI it will // reschedule anyway and update its heap key to the correct value. - sCPUHeap->ModifyKey(&sCPUEntries[targetCPU], threadPriority); + affine_update_priority_heaps(targetCPU, threadPriority); if (targetCPU == smp_get_current_cpu()) { gCPU[targetCPU].invoke_scheduler = true; @@ -391,11 +476,11 @@ affine_enqueue_in_run_queue(Thread *thread) static inline void affine_put_back(Thread* thread) { - bool pinned = sCPURunQueues != NULL && thread->pinned_to_cpu > 0; + bool pinned = sPinnedRunQueues != NULL && thread->pinned_to_cpu > 0; if (pinned) { int32 pinnedCPU = thread->previous_cpu->cpu_num; - sCPURunQueues[pinnedCPU].PushFront(thread, + sPinnedRunQueues[pinnedCPU].PushFront(thread, affine_get_effective_priority(thread)); } else { int32 previousCore = thread->scheduler_data->previous_core; @@ -492,7 +577,7 @@ affine_set_thread_priority(Thread *thread, int32 priority) affine_get_effective_priority(thread)); if (thread->state == B_THREAD_RUNNING) - sCPUHeap->ModifyKey(&sCPUEntries[thread->cpu->cpu_num], priority); + affine_update_priority_heaps(thread->cpu->cpu_num, priority); if (thread->state != B_THREAD_READY) { affine_cancel_penalty(thread); @@ -637,8 +722,8 @@ affine_dequeue_thread(int32 thisCPU) Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); Thread* pinnedThread = NULL; - if (sCPURunQueues != NULL) - pinnedThread = sCPURunQueues[thisCPU].PeekMaximum(); + if (sPinnedRunQueues != NULL) + pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); if (sharedThread == NULL && pinnedThread == NULL) return NULL; @@ -656,7 +741,7 @@ affine_dequeue_thread(int32 thisCPU) return sharedThread; } - sCPURunQueues[thisCPU].Remove(pinnedThread); + sPinnedRunQueues[thisCPU].Remove(pinnedThread); return pinnedThread; } @@ -692,7 +777,7 @@ affine_reschedule(void) // update CPU heap so that old thread would have CPU properly chosen Thread* nextThread = sRunQueues[thisCore].PeekMaximum(); if (nextThread != NULL) { - sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], + affine_update_priority_heaps(thisCPU, affine_get_effective_priority(nextThread)); } @@ -735,10 +820,10 @@ affine_reschedule(void) // select thread with the biggest priority if (oldThread->cpu->disabled) { - ASSERT(sCPURunQueues != NULL); - nextThread = sCPURunQueues[thisCPU].PeekMaximum(); + ASSERT(sPinnedRunQueues != NULL); + nextThread = sPinnedRunQueues[thisCPU].PeekMaximum(); if (nextThread != NULL) - sCPURunQueues[thisCPU].Remove(nextThread); + sPinnedRunQueues[thisCPU].Remove(nextThread); else { nextThread = sRunQueues[thisCore].GetHead(B_IDLE_PRIORITY); if (nextThread != NULL) @@ -759,7 +844,7 @@ affine_reschedule(void) oldThread, nextThread); // update CPU heap - sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], + affine_update_priority_heaps(thisCPU, affine_get_effective_priority(nextThread)); nextThread->state = B_THREAD_RUNNING; @@ -857,41 +942,12 @@ scheduler_affine_init() { int32 cpuCount = smp_get_num_cpus(); - sCPUHeap = new AffineCPUHeap; - if (sCPUHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter cpuHeapDeleter(sCPUHeap); - - sCPUEntries = new CPUHeapEntry[cpuCount]; - if (sCPUEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuEntriesDeleter(sCPUEntries); - - for (int i = 0; i < cpuCount; i++) { - sCPUEntries[i].fCPUNumber = i; - status_t result = sCPUHeap->Insert(&sCPUEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; - } - - TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-cpu queue%s\n", - cpuCount, cpuCount != 1 ? "s" : ""); - + // create logical processor to core mapping sCPUToCore = new(std::nothrow) int32[cpuCount]; if (sCPUToCore == NULL) return B_NO_MEMORY; ArrayDeleter cpuToCoreDeleter(sCPUToCore); - sCPURunQueues = new(std::nothrow) AffineRunQueue[cpuCount]; - if (sCPURunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuRunQueuesDeleter(sCPURunQueues); - for (int i = 0; i < cpuCount; i++) { - status_t result = sCPURunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - int32 coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) @@ -926,6 +982,73 @@ scheduler_affine_init() } } + // create logical processor and core heaps + sCPUPriorityEntries = new CPUHeapEntry[cpuCount]; + if (sCPUPriorityEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuPriorityEntriesDeleter(sCPUPriorityEntries); + + sCorePriorityEntries = new CoreHeapEntry[coreCount]; + if (sCorePriorityEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter corePriorityEntriesDeleter( + sCorePriorityEntries); + + sCPUPriorityHeaps = new AffineCPUHeap[coreCount]; + if (sCPUPriorityHeaps == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); + + sCPUMaxPriorityHeaps = new AffineCPUMaxHeap[coreCount]; + if (sCPUMaxPriorityHeaps == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuMaxPriorityHeapDeleter( + sCPUMaxPriorityHeaps); + + for (int32 i = 0; i < cpuCount; i++) { + sCPUPriorityEntries[i].fCPUNumber = i; + int32 core = sCPUToCore[i]; + + status_t result + = sCPUPriorityHeaps[core].Insert(&sCPUPriorityEntries[i], + B_IDLE_PRIORITY); + if (result != B_OK) + return result; + + result = sCPUMaxPriorityHeaps[core].Insert(&sCPUPriorityEntries[i], + B_IDLE_PRIORITY); + if (result != B_OK) + return result; + } + + sCorePriorityHeap = new AffineCoreHeap; + if (sCorePriorityHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter corePriorityHeapDeleter(sCorePriorityHeap); + + for (int32 i = 0; i < coreCount; i++) { + sCorePriorityEntries[i].fCoreID = i; + status_t result = sCorePriorityHeap->Insert(&sCorePriorityEntries[i], + B_IDLE_PRIORITY); + if (result != B_OK) + return result; + } + + // create per-logical processor run queues for pinned threads + TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-cpu queue%s\n", + cpuCount, cpuCount != 1 ? "s" : ""); + + sPinnedRunQueues = new(std::nothrow) AffineRunQueue[cpuCount]; + if (sPinnedRunQueues == NULL) + return B_NO_MEMORY; + ArrayDeleter pinnedRunQueuesDeleter(sPinnedRunQueues); + for (int i = 0; i < cpuCount; i++) { + status_t result = sPinnedRunQueues[i].GetInitStatus(); + if (result != B_OK) + return result; + } + + // create per-core run queues TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-core queue%s\n", coreCount, coreCount != 1 ? "s" : ""); @@ -947,10 +1070,13 @@ scheduler_affine_init() "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", 0); - cpuHeapDeleter.Detach(); - cpuEntriesDeleter.Detach(); - cpuToCoreDeleter.Detach(); - cpuRunQueuesDeleter.Detach(); runQueuesDeleter.Detach(); + pinnedRunQueuesDeleter.Detach(); + corePriorityHeapDeleter.Detach(); + cpuMaxPriorityHeapDeleter.Detach(); + cpuPriorityHeapDeleter.Detach(); + corePriorityEntriesDeleter.Detach(); + cpuPriorityEntriesDeleter.Detach(); + cpuToCoreDeleter.Detach(); return B_OK; } From 18c0d163ede074f15215e2c9caec15ae4cbd0f26 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 17 Oct 2013 19:22:29 +0200 Subject: [PATCH 049/273] kernel/util: Add MinMaxHeap implementation --- headers/private/kernel/util/MinMaxHeap.h | 478 +++++++++++++++++++++++ 1 file changed, 478 insertions(+) create mode 100644 headers/private/kernel/util/MinMaxHeap.h diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h new file mode 100644 index 0000000000..0727a85cd0 --- /dev/null +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -0,0 +1,478 @@ +/* + * Copyright 2013 Haiku, Inc. All rights reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, pdziepak@quarnos.org + */ +#ifndef KERNEL_UTIL_MIN_MAX_HEAP_H +#define KERNEL_UTIL_MIN_MAX_HEAP_H + + +#include + +#include + + +template +struct MinMaxHeapLink { + MinMaxHeapLink(); + + bool fMinTree; + int fIndex; + Key fKey; +}; + +template +class MinMaxHeapLinkImpl { +private: + typedef MinMaxHeapLink Link; + +public: + inline Link* GetMinMaxHeapLink(); + +private: + Link fMinMaxHeapLink; +}; + +template +class MinMaxHeapStandardGetLink { +private: + typedef MinMaxHeapLink Link; + +public: + inline Link* operator()(Element* element) const; +}; + +template Element::*LinkMember> +class MinMaxHeapMemberGetLink { +private: + typedef MinMaxHeapLink Link; + +public: + inline Link* operator()(Element* element) const; +}; + +template +class MinMaxHeapCompare { +public: + inline bool operator()(Key a, Key b); +}; + +#define MIN_MAX_HEAP_TEMPLATE_LIST \ + template +#define MIN_MAX_HEAP_CLASS_NAME MinMaxHeap + +template, + typename GetLink = MinMaxHeapStandardGetLink > +class MinMaxHeap { +public: + MinMaxHeap(); + ~MinMaxHeap(); + + inline Element* PeekMinimum(); + inline Element* PeekMaximum(); + + static const Key& GetKey(Element* element); + + inline void ModifyKey(Element* element, Key newKey); + + inline void RemoveMinimum(); + inline void RemoveMaximum(); + + inline status_t Insert(Element* element, Key key); + +private: + status_t _GrowHeap(); + + void _MoveUp(MinMaxHeapLink* link); + void _MoveDown(MinMaxHeapLink* link); + bool _ChangeTree(MinMaxHeapLink* link); + + void _RemoveLast(bool minTree); + + Element** fMinElements; + int fMinLastElement; + + Element** fMaxElements; + int fMaxLastElement; + + int fSize; + + static Compare sCompare; + static GetLink sGetLink; +}; + + +#if KDEBUG +template +MinMaxHeapLink::MinMaxHeapLink() + : + fIndex(-1) +{ +} +#else +template +MinMaxHeapLink::MinMaxHeapLink() +{ +} +#endif + + +template +MinMaxHeapLink* +MinMaxHeapLinkImpl::GetMinMaxHeapLink() +{ + return &fMinMaxHeapLink; +} + + +template +MinMaxHeapLink* +MinMaxHeapStandardGetLink::operator()(Element* element) const +{ + return element->GetMinMaxHeapLink(); +} + + +template Element::*LinkMember> +MinMaxHeapLink* +MinMaxHeapMemberGetLink::operator()( + Element* element) const +{ + return &(element->*LinkMember); +} + + +template +bool +MinMaxHeapCompare::operator()(Key a, Key b) +{ + return a < b; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +MIN_MAX_HEAP_CLASS_NAME::MinMaxHeap() + : + fMinElements(NULL), + fMinLastElement(0), + fMaxElements(NULL), + fMaxLastElement(0), + fSize(0) +{ +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap() +{ + free(fMinElements); + free(fMaxElements); +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +Element* +MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() +{ + if (fMinLastElement > 0) + return fMinElements[0]; + else if (fMaxLastElement > 0) { + ASSERT(fMaxLastElement == 1); + return fMaxElements[0]; + } + + return NULL; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +Element* +MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() +{ + if (fMaxLastElement > 0) + return fMaxElements[0]; + else if (fMinLastElement > 0) { + ASSERT(fMinLastElement == 1); + return fMinElements[0]; + } + + return NULL; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +const Key& +MIN_MAX_HEAP_CLASS_NAME::GetKey(Element* element) +{ + return sGetLink(element)->fKey; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +void +MIN_MAX_HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey) +{ + MinMaxHeapLink* link = sGetLink(element); + + Key oldKey = link->fKey; + link->fKey = newKey; + + if (sCompare(newKey, oldKey) && link->fMinTree) + _MoveUp(link); + else + _MoveDown(link); +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +void +MIN_MAX_HEAP_CLASS_NAME::RemoveMinimum() +{ + if (fMinLastElement == 0) { + ASSERT(fMaxLastElement == 1); + RemoveMaximum(); + return; + } + +#if KDEBUG + Element* element = PeekMinimum(); + MinMaxHeapLink* link = sGetLink(element); + link->fIndex = -1; +#endif + + _RemoveLast(true); +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +void +MIN_MAX_HEAP_CLASS_NAME::RemoveMaximum() +{ + if (fMaxLastElement == 0) { + ASSERT(fMinLastElement == 1); + RemoveMinimum(); + return; + } + +#if KDEBUG + Element* element = PeekMaximum(); + MinMaxHeapLink* link = sGetLink(element); + link->fIndex = -1; +#endif + + _RemoveLast(false); +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +status_t +MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key) +{ + + if (min_c(fMinLastElement, fMaxLastElement) == fSize) { + ASSERT(max_c(fMinLastElement, fMaxLastElement) == fSize); + status_t result = _GrowHeap(); + if (result != B_OK) + return result; + } + + ASSERT(fMinLastElement != fSize || fMaxLastElement != fSize); + + MinMaxHeapLink* link = sGetLink(element); + + link->fMinTree = fMinLastElement < fMaxLastElement; + + int& lastElement = link->fMinTree ? fMinLastElement : fMaxLastElement; + Element** tree = link->fMinTree ? fMinElements : fMaxElements; + + tree[lastElement] = element; + link->fIndex = lastElement++; + link->fKey = key; + + if (!_ChangeTree(link)) + _MoveUp(link); + + return B_OK; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +status_t +MIN_MAX_HEAP_CLASS_NAME::_GrowHeap() +{ + int newSize = max_c(fSize * 2, 4); + + size_t arraySize = newSize * sizeof(Element*); + Element** newBuffer + = reinterpret_cast(realloc(fMinElements, arraySize)); + if (newBuffer == NULL) + return B_NO_MEMORY; + fMinElements = newBuffer; + + newBuffer + = reinterpret_cast(realloc(fMaxElements, arraySize)); + if (newBuffer == NULL) + return B_NO_MEMORY; + fMaxElements = newBuffer; + + fSize = newSize; + return B_OK; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +void +MIN_MAX_HEAP_CLASS_NAME::_MoveUp(MinMaxHeapLink* link) +{ + Element** tree = link->fMinTree ? fMinElements : fMaxElements; + while (true) { + if (link->fIndex <= 0) + break; + + int parent = (link->fIndex - 1) / 2; + bool isSmaller = sCompare(link->fKey, sGetLink(tree[parent])->fKey); + if (isSmaller ^ !link->fMinTree) { + ASSERT(sGetLink(tree[parent])->fIndex == parent); + sGetLink(tree[parent])->fIndex = link->fIndex; + + Element* element = tree[link->fIndex]; + tree[link->fIndex] = tree[parent]; + tree[parent] = element; + + link->fIndex = parent; + } else + break; + } +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +void +MIN_MAX_HEAP_CLASS_NAME::_MoveDown(MinMaxHeapLink* link) +{ + int current; + + int lastElement = link->fMinTree ? fMinLastElement : fMaxLastElement; + Element** tree = link->fMinTree ? fMinElements : fMaxElements; + while (true) { + current = link->fIndex; + + int child = 2 * link->fIndex + 1; + if (child < lastElement) { + bool isSmaller = sCompare(sGetLink(tree[child])->fKey, link->fKey); + if (isSmaller ^ !link->fMinTree) + current = child; + } + + child = 2 * link->fIndex + 2; + if (child < lastElement) { + bool isSmaller = sCompare(sGetLink(tree[child])->fKey, + sGetLink(tree[current])->fKey); + if (isSmaller ^ !link->fMinTree) + current = child; + } + + if (link->fIndex == current) + break; + + ASSERT(sGetLink(tree[current])->fIndex == current); + sGetLink(tree[current])->fIndex = link->fIndex; + + Element* element = tree[link->fIndex]; + tree[link->fIndex] = tree[current]; + tree[current] = element; + + link->fIndex = current; + } + + if (2 * link->fIndex + 1 >= lastElement) + _ChangeTree(link); +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +bool +MIN_MAX_HEAP_CLASS_NAME::_ChangeTree(MinMaxHeapLink* link) +{ + int currentLastElement = link->fMinTree ? fMinLastElement : fMaxLastElement; + int otherLastElement = link->fMinTree ? fMaxLastElement : fMinLastElement; + + Element** currentTree = link->fMinTree ? fMinElements : fMaxElements; + Element** otherTree = link->fMinTree ? fMaxElements : fMinElements; + + if (otherLastElement <= 0) { + ASSERT(currentLastElement == 1); + return true; + } + + ASSERT((link->fIndex - 1) / 2 < otherLastElement); + + Element* predecessor; + if (2 * link->fIndex + 1 < otherLastElement) { + predecessor = otherTree[2 * link->fIndex + 1]; + ASSERT(sGetLink(predecessor)->fIndex == 2 * link->fIndex + 1); + } else if (link->fIndex < otherLastElement) { + predecessor = otherTree[link->fIndex]; + ASSERT(sGetLink(predecessor)->fIndex == link->fIndex); + } else { + predecessor = otherTree[(link->fIndex - 1) / 2]; + ASSERT(sGetLink(predecessor)->fIndex == (link->fIndex - 1) / 2); + } + MinMaxHeapLink* predecessorLink = sGetLink(predecessor); + + bool isSmaller = sCompare(predecessorLink->fKey, link->fKey); + if (isSmaller ^ !link->fMinTree) { + Element* element = currentTree[link->fIndex]; + currentTree[link->fIndex] = otherTree[predecessorLink->fIndex]; + otherTree[predecessorLink->fIndex] = element; + + int index = link->fIndex; + link->fIndex = predecessorLink->fIndex; + predecessorLink->fIndex = index; + + predecessorLink->fMinTree = !predecessorLink->fMinTree; + link->fMinTree = !link->fMinTree; + + _MoveUp(link); + return true; + } + + return false; +} + + +MIN_MAX_HEAP_TEMPLATE_LIST +void +MIN_MAX_HEAP_CLASS_NAME::_RemoveLast(bool minTree) +{ + bool deleteMin = fMaxLastElement < fMinLastElement; + + Element** tree = deleteMin ? fMinElements : fMaxElements; + int& lastElement = deleteMin ? fMinLastElement : fMaxLastElement; + + ASSERT(lastElement > 0); + lastElement--; + if (lastElement == 0 && deleteMin == minTree) + return; + + Element* element = tree[lastElement]; + + if (minTree) + fMinElements[0] = element; + else + fMaxElements[0] = element; + + MinMaxHeapLink* link = sGetLink(element); + link->fIndex = 0; + link->fMinTree = minTree; + _MoveDown(link); +} + + +#endif // KERNEL_UTIL_MIN_MAX_HEAP_H + From da3a48f4a8a723322ad53f06fddf945a7c38ea3d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 17 Oct 2013 19:23:27 +0200 Subject: [PATCH 050/273] scheduler_affine: Use min-max heap as per-core CPU heap --- .../kernel/scheduler/scheduler_affine.cpp | 41 +++++-------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index b1667bf07b..1dbcc6f132 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "RunQueue.h" @@ -48,20 +49,13 @@ const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; -struct CPUHeapEntry : public HeapLinkImpl { - HeapLink fMaxHeap; +struct CPUHeapEntry : public MinMaxHeapLinkImpl { int32 fCPUNumber; }; static CPUHeapEntry* sCPUPriorityEntries; -typedef Heap AffineCPUHeap; -typedef Heap, - HeapMemberGetLink > - AffineCPUMaxHeap; - -// TODO: Use one min-max heap per-core +typedef MinMaxHeap AffineCPUHeap; static AffineCPUHeap* sCPUPriorityHeaps; -static AffineCPUMaxHeap* sCPUMaxPriorityHeaps; struct CoreHeapEntry : public HeapLinkImpl { int32 fCoreID; @@ -214,25 +208,25 @@ dump_heap(AffineCPUHeap* heap) AffineCPUHeap temp; kprintf("cpu priority actual priority\n"); - CPUHeapEntry* entry = heap->PeekRoot(); + CPUHeapEntry* entry = heap->PeekMinimum(); while (entry) { int32 cpu = entry->fCPUNumber; int32 key = AffineCPUHeap::GetKey(entry); kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, key, affine_get_effective_priority(gCPU[cpu].running_thread)); - heap->RemoveRoot(); + heap->RemoveMinimum(); temp.Insert(entry, key); - entry = heap->PeekRoot(); + entry = heap->PeekMinimum(); } - entry = temp.PeekRoot(); + entry = temp.PeekMinimum(); while (entry) { int32 key = AffineCPUHeap::GetKey(entry); - temp.RemoveRoot(); + temp.RemoveMinimum(); heap->Insert(entry, key); - entry = temp.PeekRoot(); + entry = temp.PeekMinimum(); } } @@ -355,10 +349,9 @@ affine_update_priority_heaps(int32 cpu, int32 priority) int32 core = sCPUToCore[cpu]; sCPUPriorityHeaps[core].ModifyKey(&sCPUPriorityEntries[cpu], priority); - sCPUMaxPriorityHeaps[core].ModifyKey(&sCPUPriorityEntries[cpu], priority); int32 maxPriority - = AffineCPUMaxHeap::GetKey(sCPUMaxPriorityHeaps[core].PeekRoot()); + = AffineCPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); int32 corePriority = AffineCoreHeap::GetKey(&sCorePriorityEntries[core]); if (corePriority != maxPriority) @@ -378,7 +371,7 @@ affine_choose_core(void) static inline int32 affine_choose_cpu(int32 core) { - CPUHeapEntry* entry = sCPUPriorityHeaps[core].PeekRoot(); + CPUHeapEntry* entry = sCPUPriorityHeaps[core].PeekMinimum(); ASSERT(entry != NULL); return entry->fCPUNumber; } @@ -999,12 +992,6 @@ scheduler_affine_init() return B_NO_MEMORY; ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); - sCPUMaxPriorityHeaps = new AffineCPUMaxHeap[coreCount]; - if (sCPUMaxPriorityHeaps == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuMaxPriorityHeapDeleter( - sCPUMaxPriorityHeaps); - for (int32 i = 0; i < cpuCount; i++) { sCPUPriorityEntries[i].fCPUNumber = i; int32 core = sCPUToCore[i]; @@ -1014,11 +1001,6 @@ scheduler_affine_init() B_IDLE_PRIORITY); if (result != B_OK) return result; - - result = sCPUMaxPriorityHeaps[core].Insert(&sCPUPriorityEntries[i], - B_IDLE_PRIORITY); - if (result != B_OK) - return result; } sCorePriorityHeap = new AffineCoreHeap; @@ -1073,7 +1055,6 @@ scheduler_affine_init() runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); corePriorityHeapDeleter.Detach(); - cpuMaxPriorityHeapDeleter.Detach(); cpuPriorityHeapDeleter.Detach(); corePriorityEntriesDeleter.Detach(); cpuPriorityEntriesDeleter.Detach(); From a6d4233e59a5c4b9446fb12e2eec4fb2b399703f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 20 Oct 2013 23:26:32 +0200 Subject: [PATCH 051/273] scheduler_affine: Choose wisely which core to wake up The longer core is idle the deeper idle state it has entered. That's why the scheduler should always choose the core that has gone idle most recently (both for performance and power saving reasons). Moreover, if there are more than one package the scheduler should minimize the number of packages with at least one core active when power saving is the priority. Contrary, as many packages as possible should be used when aiming for high performance. --- .../kernel/scheduler/scheduler_affine.cpp | 344 +++++++++++++++--- 1 file changed, 298 insertions(+), 46 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 1dbcc6f132..c8cb8ea6c7 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2013. Paweł Dziepak, pdziepak@quarnos.org. + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2009, Rene Gollent, rene@gollent.com. * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. @@ -49,22 +49,52 @@ const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; -struct CPUHeapEntry : public MinMaxHeapLinkImpl { +// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical +// processors in the heap belong to. Since there are no cache affinity issues +// at this level and the run queue is shared among all logical processors on +// the core the only real concern is to make lower priority threads give way to +// the higher priority threads. +struct CPUEntry : public MinMaxHeapLinkImpl { int32 fCPUNumber; }; - -static CPUHeapEntry* sCPUPriorityEntries; -typedef MinMaxHeap AffineCPUHeap; +typedef MinMaxHeap AffineCPUHeap; +static CPUEntry* sCPUEntries; static AffineCPUHeap* sCPUPriorityHeaps; -struct CoreHeapEntry : public HeapLinkImpl { +struct CoreEntry : public HeapLinkImpl, + DoublyLinkedListLinkImpl { int32 fCoreID; }; -static CoreHeapEntry* sCorePriorityEntries; -typedef Heap AffineCoreHeap; +static CoreEntry* sCoreEntries; +typedef Heap AffineCoreHeap; static AffineCoreHeap* sCorePriorityHeap; +// sPackageUsageHeap is used to decide which core should be woken up from the +// idle state. When aiming for performance we should use as many packages as +// possible with as little cores active in each package as possible (so that the +// package can enter any boost mode if it has one and the active core have more +// of the shared cache for themselves. If power saving is the main priority we +// should keep active cores on as little packages as possible (so that other +// packages can go to the deep state of sleep). The heap stores only packages +// with at least one core active and one core idle. The packages with all cores +// idle are stored in sPackageIdleList (in LIFO manner). +struct PackageEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { + int32 fPackageID; + + DoublyLinkedList fIdleCores; + int32 fIdleCoreCount; + + int32 fCoreCount; +}; +typedef MinMaxHeap AffinePackageHeap; +typedef DoublyLinkedList AffineIdlePackageList; + +static PackageEntry* sPackageEntries; +static AffinePackageHeap* sPackageUsageHeap; +static AffineIdlePackageList* sIdlePackageList; + // The run queues. Holds the threads ready to run ordered by priority. // One queue per schedulable target per core. Additionally, each // logical processor has its sPinnedRunQueues used for scheduling @@ -73,7 +103,12 @@ typedef RunQueue AffineRunQueue; static AffineRunQueue* sRunQueues; static AffineRunQueue* sPinnedRunQueues; static int32 sRunQueueCount; + +// Since CPU IDs used internally by the kernel bear no relation to the actual +// CPU topology the following arrays are used to efficiently get the core +// and the package that CPU in question belongs to. static int32* sCPUToCore; +static int32* sCPUToPackage; struct scheduler_thread_data { @@ -133,7 +168,6 @@ affine_get_thread_penalty(Thread* thread) } return penalty; - } @@ -208,7 +242,7 @@ dump_heap(AffineCPUHeap* heap) AffineCPUHeap temp; kprintf("cpu priority actual priority\n"); - CPUHeapEntry* entry = heap->PeekMinimum(); + CPUEntry* entry = heap->PeekMinimum(); while (entry) { int32 cpu = entry->fCPUNumber; int32 key = AffineCPUHeap::GetKey(entry); @@ -237,7 +271,7 @@ dump_cpu_heap(int argc, char** argv) AffineCoreHeap temp; kprintf("core priority\n"); - CoreHeapEntry* entry = sCorePriorityHeap->PeekRoot(); + CoreEntry* entry = sCorePriorityHeap->PeekRoot(); while (entry) { int32 core = entry->fCoreID; int32 key = AffineCoreHeap::GetKey(entry); @@ -266,6 +300,79 @@ dump_cpu_heap(int argc, char** argv) } +static int +dump_idle_cores(int argc, char** argv) +{ + kprintf("Idle packages:\n"); + AffineIdlePackageList::ReverseIterator idleIterator + = sIdlePackageList->GetReverseIterator(); + + if (idleIterator.HasNext()) { + kprintf("package cores\n"); + + while (idleIterator.HasNext()) { + PackageEntry* entry = idleIterator.Next(); + kprintf("%-7" B_PRId32 " ", entry->fPackageID); + + DoublyLinkedList::ReverseIterator iterator + = entry->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); + } + } else + kprintf("No idle packages.\n"); + + AffinePackageHeap temp; + temp.GrowHeap(smp_get_num_cpus()); + kprintf("\nPackages with idle cores:\n"); + + PackageEntry* entry = sPackageUsageHeap->PeekMinimum(); + if (entry == NULL) + kprintf("No packages.\n"); + else + kprintf("package count cores\n"); + + while (entry != NULL) { + kprintf("%-7" B_PRId32 " %-5" B_PRId32 " ", entry->fPackageID, + entry->fIdleCoreCount); + + DoublyLinkedList::ReverseIterator iterator + = entry->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); + + sPackageUsageHeap->RemoveMinimum(); + temp.Insert(entry, entry->fIdleCoreCount); + + entry = sPackageUsageHeap->PeekMinimum(); + } + + entry = temp.PeekMinimum(); + while (entry != NULL) { + int32 key = AffinePackageHeap::GetKey(entry); + temp.RemoveMinimum(); + sPackageUsageHeap->Insert(entry, key); + entry = temp.PeekMinimum(); + } + + return 0; +} + + static void affine_dump_thread_data(Thread* thread) { @@ -348,21 +455,92 @@ affine_update_priority_heaps(int32 cpu, int32 priority) { int32 core = sCPUToCore[cpu]; - sCPUPriorityHeaps[core].ModifyKey(&sCPUPriorityEntries[cpu], priority); + sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); int32 maxPriority = AffineCPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); - int32 corePriority = AffineCoreHeap::GetKey(&sCorePriorityEntries[core]); + int32 corePriority = AffineCoreHeap::GetKey(&sCoreEntries[core]); - if (corePriority != maxPriority) - sCorePriorityHeap->ModifyKey(&sCorePriorityEntries[core], maxPriority); + if (corePriority != maxPriority) { + sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); + + int32 package = sCPUToPackage[cpu]; + PackageEntry* packageEntry = &sPackageEntries[package]; + if (maxPriority == B_IDLE_PRIORITY) { + // core goes idle + ASSERT(packageEntry->fIdleCoreCount >= 0); + ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount++; + packageEntry->fIdleCores.Add(&sCoreEntries[core]); + + if (packageEntry->fIdleCoreCount == 1) { + // first core on that package to go idle + + if (packageEntry->fCoreCount > 1) + sPackageUsageHeap->Insert(packageEntry, 1); + else + sIdlePackageList->Add(packageEntry); + } else if (packageEntry->fIdleCoreCount + == packageEntry->fCoreCount) { + // package goes idle + sPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); + sPackageUsageHeap->RemoveMinimum(); + + sIdlePackageList->Add(packageEntry); + } else { + sPackageUsageHeap->ModifyKey(packageEntry, + packageEntry->fIdleCoreCount); + } + } else if (corePriority == B_IDLE_PRIORITY) { + // core wakes up + ASSERT(packageEntry->fIdleCoreCount > 0); + ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount--; + packageEntry->fIdleCores.Remove(&sCoreEntries[core]); + + if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { + // package wakes up + sIdlePackageList->Remove(packageEntry); + + if (packageEntry->fIdleCoreCount > 0) { + sPackageUsageHeap->Insert(packageEntry, + packageEntry->fIdleCoreCount); + } + } else if (packageEntry->fIdleCoreCount == 0) { + // no more idle cores in the package + sPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); + sPackageUsageHeap->RemoveMinimum(); + } else { + sPackageUsageHeap->ModifyKey(packageEntry, + packageEntry->fIdleCoreCount); + } + } + } } static inline int32 affine_choose_core(void) { - CoreHeapEntry* entry = sCorePriorityHeap->PeekRoot(); + CoreEntry* entry; + + if (sIdlePackageList->Last() != NULL) { + // wake new package + PackageEntry* package = sIdlePackageList->Last(); + entry = package->fIdleCores.Last(); + } else if (sPackageUsageHeap->PeekMaximum() != NULL) { + // wake new core + PackageEntry* package = sPackageUsageHeap->PeekMaximum(); + entry = package->fIdleCores.Last(); + } else { + // no idle cores, use least occupied core + entry = sCorePriorityHeap->PeekRoot(); + } + ASSERT(entry != NULL); return entry->fCoreID; } @@ -371,7 +549,7 @@ affine_choose_core(void) static inline int32 affine_choose_cpu(int32 core) { - CPUHeapEntry* entry = sCPUPriorityHeaps[core].PeekMinimum(); + CPUEntry* entry = sCPUPriorityHeaps[core].PeekMinimum(); ASSERT(entry != NULL); return entry->fCPUNumber; } @@ -935,12 +1113,17 @@ scheduler_affine_init() { int32 cpuCount = smp_get_num_cpus(); - // create logical processor to core mapping + // create logical processor to core and package mappings sCPUToCore = new(std::nothrow) int32[cpuCount]; if (sCPUToCore == NULL) return B_NO_MEMORY; ArrayDeleter cpuToCoreDeleter(sCPUToCore); + sCPUToPackage = new(std::nothrow) int32[cpuCount]; + if (sCPUToPackage == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuToPackageDeleter(sCPUToPackage); + int32 coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) @@ -948,6 +1131,14 @@ scheduler_affine_init() } sRunQueueCount = coreCount; + int32 packageCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 + && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { + sCPUToPackage[i] = packageCount++; + } + } + // TODO: Nasty O(n^2), solutions with better complexity will require // creating helper data structures. This code is run only once, so it // probably won't be a problem until we support systems with large @@ -975,47 +1166,102 @@ scheduler_affine_init() } } - // create logical processor and core heaps - sCPUPriorityEntries = new CPUHeapEntry[cpuCount]; - if (sCPUPriorityEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuPriorityEntriesDeleter(sCPUPriorityEntries); - - sCorePriorityEntries = new CoreHeapEntry[coreCount]; - if (sCorePriorityEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter corePriorityEntriesDeleter( - sCorePriorityEntries); - - sCPUPriorityHeaps = new AffineCPUHeap[coreCount]; - if (sCPUPriorityHeaps == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); - + // TODO: Another O(n^2), something has to be done with that... (i.e. + // build a tree representing the topology and then use it to create + // these mappings. for (int32 i = 0; i < cpuCount; i++) { - sCPUPriorityEntries[i].fCPUNumber = i; - int32 core = sCPUToCore[i]; + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 + && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { + continue; + } - status_t result - = sCPUPriorityHeaps[core].Insert(&sCPUPriorityEntries[i], - B_IDLE_PRIORITY); - if (result != B_OK) - return result; + for (int32 j = 0; j < cpuCount; j++) { + bool samePackage = true; + for (int32 k = 0; k < CPU_TOPOLOGY_LEVELS && samePackage; k++) { + if (k < CPU_TOPOLOGY_PACKAGE) { + if (gCPU[j].topology_id[k] == 0) + continue; + samePackage = false; + } + + samePackage = gCPU[i].topology_id[k] == gCPU[j].topology_id[k]; + } + + if (samePackage) + sCPUToPackage[i] = sCPUToPackage[j]; + } } + // create package heap and idle package stack + sPackageEntries = new(std::nothrow) PackageEntry[packageCount]; + if (sPackageEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter packageEntriesDeleter(sPackageEntries); + + sPackageUsageHeap = new(std::nothrow) AffinePackageHeap; + if (sPackageUsageHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter packageHeapDeleter(sPackageUsageHeap); + status_t result = sPackageUsageHeap->GrowHeap(packageCount); + if (result != B_OK) + return B_OK; + + sIdlePackageList = new(std::nothrow) AffineIdlePackageList; + if (sIdlePackageList == NULL) + return B_NO_MEMORY; + ObjectDeleter packageListDeleter(sIdlePackageList); + + for (int32 i = 0; i < packageCount; i++) { + sPackageEntries[i].fPackageID = i; + sPackageEntries[i].fIdleCoreCount = coreCount / packageCount; + sPackageEntries[i].fCoreCount = coreCount / packageCount; + sIdlePackageList->Insert(&sPackageEntries[i]); + } + + // create logical processor and core heaps + sCPUEntries = new CPUEntry[cpuCount]; + if (sCPUEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuEntriesDeleter(sCPUEntries); + + sCoreEntries = new CoreEntry[coreCount]; + if (sCoreEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter coreEntriesDeleter( + sCoreEntries); + sCorePriorityHeap = new AffineCoreHeap; if (sCorePriorityHeap == NULL) return B_NO_MEMORY; ObjectDeleter corePriorityHeapDeleter(sCorePriorityHeap); for (int32 i = 0; i < coreCount; i++) { - sCorePriorityEntries[i].fCoreID = i; - status_t result = sCorePriorityHeap->Insert(&sCorePriorityEntries[i], + sCoreEntries[i].fCoreID = i; + status_t result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; } + sCPUPriorityHeaps = new AffineCPUHeap[coreCount]; + if (sCPUPriorityHeaps == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); + + for (int32 i = 0; i < cpuCount; i++) { + sCPUEntries[i].fCPUNumber = i; + int32 core = sCPUToCore[i]; + + int32 package = sCPUToPackage[i]; + if (sCPUPriorityHeaps[core].PeekMaximum() == NULL) + sPackageEntries[package].fIdleCores.Insert(&sCoreEntries[core]); + + status_t result + = sCPUPriorityHeaps[core].Insert(&sCPUEntries[i], B_IDLE_PRIORITY); + if (result != B_OK) + return result; + } + // create per-logical processor run queues for pinned threads TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-cpu queue%s\n", cpuCount, cpuCount != 1 ? "s" : ""); @@ -1051,13 +1297,19 @@ scheduler_affine_init() add_debugger_command_etc("cpu_heap", &dump_cpu_heap, "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", 0); + add_debugger_command_etc("idle_cores", &dump_idle_cores, + "List idle cores", "\nList idle cores", 0); runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); corePriorityHeapDeleter.Detach(); cpuPriorityHeapDeleter.Detach(); - corePriorityEntriesDeleter.Detach(); - cpuPriorityEntriesDeleter.Detach(); + coreEntriesDeleter.Detach(); + cpuEntriesDeleter.Detach(); + packageEntriesDeleter.Detach(); + packageHeapDeleter.Detach(); + packageListDeleter.Detach(); + cpuToPackageDeleter.Detach(); cpuToCoreDeleter.Detach(); return B_OK; } From 5cbf227236afa227e172a3ea7d5127a9ab0770dc Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 20 Oct 2013 23:33:55 +0200 Subject: [PATCH 052/273] kernel/util: Allocate only one array in MinMaxHeap --- headers/private/kernel/util/MinMaxHeap.h | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 0727a85cd0..01a48d1be9 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -84,9 +84,8 @@ public: inline status_t Insert(Element* element, Key key); + status_t GrowHeap(int minimalSize = 0); private: - status_t _GrowHeap(); - void _MoveUp(MinMaxHeapLink* link); void _MoveDown(MinMaxHeapLink* link); bool _ChangeTree(MinMaxHeapLink* link); @@ -171,7 +170,6 @@ MIN_MAX_HEAP_TEMPLATE_LIST MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap() { free(fMinElements); - free(fMaxElements); } @@ -273,10 +271,9 @@ MIN_MAX_HEAP_TEMPLATE_LIST status_t MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key) { - if (min_c(fMinLastElement, fMaxLastElement) == fSize) { ASSERT(max_c(fMinLastElement, fMaxLastElement) == fSize); - status_t result = _GrowHeap(); + status_t result = GrowHeap(); if (result != B_OK) return result; } @@ -303,9 +300,10 @@ MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key) MIN_MAX_HEAP_TEMPLATE_LIST status_t -MIN_MAX_HEAP_CLASS_NAME::_GrowHeap() +MIN_MAX_HEAP_CLASS_NAME::GrowHeap(int minimalSize) { - int newSize = max_c(fSize * 2, 4); + minimalSize = minimalSize % 2 ? minimalSize : minimalSize + 1; + int newSize = max_c(max_c(fSize * 4, 4), minimalSize); size_t arraySize = newSize * sizeof(Element*); Element** newBuffer @@ -313,14 +311,9 @@ MIN_MAX_HEAP_CLASS_NAME::_GrowHeap() if (newBuffer == NULL) return B_NO_MEMORY; fMinElements = newBuffer; + fMaxElements = newBuffer + (newSize / 2); - newBuffer - = reinterpret_cast(realloc(fMaxElements, arraySize)); - if (newBuffer == NULL) - return B_NO_MEMORY; - fMaxElements = newBuffer; - - fSize = newSize; + fSize = newSize / 2; return B_OK; } From 343c4896892733387f1e1b1ed01f151107e7da5f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 01:33:35 +0200 Subject: [PATCH 053/273] kernel: Create CPU topology tree --- headers/private/kernel/cpu.h | 12 ++++ src/system/kernel/cpu.cpp | 111 +++++++++++++++++++++++++++++++++++ src/system/kernel/main.cpp | 1 + 3 files changed, 124 insertions(+) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index e8fe6bc490..d3cf0a8bcd 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -39,6 +39,15 @@ typedef enum cpu_topology_level { CPU_TOPOLOGY_LEVELS } cpu_topology_level; +typedef struct cpu_topology_node { + cpu_topology_level level; + + int id; + + cpu_topology_node** children; + int children_count; +} cpu_topology_node; + /* CPU local data structure */ @@ -93,6 +102,9 @@ bigtime_t cpu_get_active_time(int32 cpu); cpu_ent *get_cpu_struct(void); extern inline cpu_ent *get_cpu_struct(void) { return &gCPU[smp_get_current_cpu()]; } +status_t cpu_build_topology_tree(void); +cpu_topology_node* get_cpu_topology(void); + void _user_clear_caches(void *address, size_t length, uint32 flags); bool _user_cpu_enabled(int32 cpu); status_t _user_set_cpu_enabled(int32 cpu, bool enabled); diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 1dfbc4dcc1..36df9aa7cc 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -21,7 +21,9 @@ /* global per-cpu structure */ cpu_ent gCPU[MAX_BOOT_CPUS]; + uint32 gCPUCacheLevelCount; +static cpu_topology_node sCPUTopology; static spinlock sSetCpuLock; @@ -88,6 +90,115 @@ clear_caches(void *address, size_t length, uint32 flags) } +static status_t +cpu_create_topology_node(cpu_topology_node* node, int32* maxID, int32 id) +{ + cpu_topology_level level = static_cast(node->level - 1); + ASSERT(level >= 0); + + cpu_topology_node* newNode = new(std::nothrow) cpu_topology_node; + if (newNode == NULL) + return B_NO_MEMORY; + node->children[id] = newNode; + + newNode->level = level; + if (level != CPU_TOPOLOGY_SMT) { + newNode->children_count = maxID[level - 1]; + newNode->children + = new(std::nothrow) cpu_topology_node*[maxID[level - 1]]; + if (newNode->children == NULL) + return B_NO_MEMORY; + + memset(newNode->children, 0, + maxID[level - 1] * sizeof(cpu_topology_node*)); + } else { + newNode->children_count = 0; + newNode->children = NULL; + } + + return B_OK; +} + + +static void +cpu_rebuild_topology_tree(cpu_topology_node* node, int32* lastID) +{ + if (node->children == NULL) + return; + + int32 count = 0; + for (int32 i = 0; i < node->children_count; i++) { + if (node->children[i] == NULL) + continue; + + if (count != i) + node->children[count] = node->children[i]; + + if (node->children[count]->level != CPU_TOPOLOGY_SMT) + node->children[count]->id = lastID[node->children[count]->level]++; + + cpu_rebuild_topology_tree(node->children[count], lastID); + count++; + } + node->children_count = count; +} + + +status_t +cpu_build_topology_tree(void) +{ + sCPUTopology.level = CPU_TOPOLOGY_LEVELS; + + int32 maxID[CPU_TOPOLOGY_LEVELS]; + memset(&maxID, 0, sizeof(maxID)); + + const int32 kCPUCount = smp_get_num_cpus(); + for (int32 i = 0; i < kCPUCount; i++) { + for (int32 j = 0; j < CPU_TOPOLOGY_LEVELS; j++) + maxID[j] = max_c(maxID[j], gCPU[i].topology_id[j]); + } + + for (int32 j = 0; j < CPU_TOPOLOGY_LEVELS; j++) + maxID[j]++; + + sCPUTopology.children_count = maxID[CPU_TOPOLOGY_LEVELS - 1]; + sCPUTopology.children + = new(std::nothrow) cpu_topology_node*[maxID[CPU_TOPOLOGY_LEVELS - 1]]; + if (sCPUTopology.children == NULL) + return B_NO_MEMORY; + memset(sCPUTopology.children, 0, + maxID[CPU_TOPOLOGY_LEVELS - 1] * sizeof(cpu_topology_node*)); + + for (int32 i = 0; i < kCPUCount; i++) { + cpu_topology_node* node = &sCPUTopology; + for (int32 j = CPU_TOPOLOGY_LEVELS - 1; j >= 0; j--) { + int32 id = gCPU[i].topology_id[j]; + if (node->children[id] == NULL) { + status_t result = cpu_create_topology_node(node, maxID, id); + if (result != B_OK) + return result; + } + + node = node->children[id]; + } + + ASSERT(node->level == CPU_TOPOLOGY_SMT); + node->id = i; + } + + int32 lastID[CPU_TOPOLOGY_LEVELS]; + memset(&lastID, 0, sizeof(lastID)); + cpu_rebuild_topology_tree(&sCPUTopology, lastID); +} + + +cpu_topology_node* +get_cpu_topology(void) +{ + return &sCPUTopology; +} + + // #pragma mark - diff --git a/src/system/kernel/main.cpp b/src/system/kernel/main.cpp index 938b3a61a6..9dedf47db3 100644 --- a/src/system/kernel/main.cpp +++ b/src/system/kernel/main.cpp @@ -165,6 +165,7 @@ _start(kernel_args *bootKernelArgs, int currentCPU) TRACE("init SMP\n"); smp_init(&sKernelArgs); + cpu_build_topology_tree(); TRACE("init timer\n"); timer_init(&sKernelArgs); TRACE("init real time clock\n"); From 4b446279e6d85e37a909856a3a6d747170b22ed8 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 01:34:31 +0200 Subject: [PATCH 054/273] scheduler_affine: Use CPU topology tree to create ID mappings --- .../kernel/scheduler/scheduler_affine.cpp | 126 ++++++++---------- 1 file changed, 56 insertions(+), 70 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index c8cb8ea6c7..4f4c1be9b8 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -1108,12 +1108,34 @@ static scheduler_ops kAffineOps = { // #pragma mark - -status_t -scheduler_affine_init() +static void +traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) { - int32 cpuCount = smp_get_num_cpus(); + switch (node->level) { + case CPU_TOPOLOGY_SMT: + sCPUToCore[node->id] = coreID; + sCPUToPackage[node->id] = packageID; + return; + + case CPU_TOPOLOGY_CORE: + coreID = node->id; + break; + + case CPU_TOPOLOGY_PACKAGE: + packageID = node->id; + break; + } + + for (int32 i = 0; i < node->children_count; i++) + traverse_topology_tree(node->children[i], packageID, coreID); +} + + +static status_t +build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) +{ + cpuCount = smp_get_num_cpus(); - // create logical processor to core and package mappings sCPUToCore = new(std::nothrow) int32[cpuCount]; if (sCPUToCore == NULL) return B_NO_MEMORY; @@ -1124,74 +1146,40 @@ scheduler_affine_init() return B_NO_MEMORY; ArrayDeleter cpuToPackageDeleter(sCPUToPackage); - int32 coreCount = 0; + coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) - sCPUToCore[i] = coreCount++; + coreCount++; } + + packageCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 + && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { + packageCount++; + } + } + + cpu_topology_node* root = get_cpu_topology(); + traverse_topology_tree(root, 0, 0); + + cpuToCoreDeleter.Detach(); + cpuToPackageDeleter.Detach(); + return B_OK; +} + + +status_t +scheduler_affine_init() +{ + // create logical processor to core and package mappings + int32 cpuCount, coreCount, packageCount; + status_t result = build_topology_mappings(cpuCount, coreCount, + packageCount); + if (result != B_OK) + return result; sRunQueueCount = coreCount; - int32 packageCount = 0; - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 - && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { - sCPUToPackage[i] = packageCount++; - } - } - - // TODO: Nasty O(n^2), solutions with better complexity will require - // creating helper data structures. This code is run only once, so it - // probably won't be a problem until we support systems with large - // number of processors. - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) - continue; - - for (int32 j = 0; j < cpuCount; j++) { - bool sameCore = true; - for (int32 k = 0; k < CPU_TOPOLOGY_LEVELS && sameCore; k++) { - if (k == CPU_TOPOLOGY_SMT && gCPU[j].topology_id[k] == 0) - continue; - if (k == CPU_TOPOLOGY_SMT && gCPU[j].topology_id[k] != 0) { - sameCore = false; - continue; - } - - if (gCPU[i].topology_id[k] != gCPU[j].topology_id[k]) - sameCore = false; - } - - if (sameCore) - sCPUToCore[i] = sCPUToCore[j]; - } - } - - // TODO: Another O(n^2), something has to be done with that... (i.e. - // build a tree representing the topology and then use it to create - // these mappings. - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 - && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { - continue; - } - - for (int32 j = 0; j < cpuCount; j++) { - bool samePackage = true; - for (int32 k = 0; k < CPU_TOPOLOGY_LEVELS && samePackage; k++) { - if (k < CPU_TOPOLOGY_PACKAGE) { - if (gCPU[j].topology_id[k] == 0) - continue; - samePackage = false; - } - - samePackage = gCPU[i].topology_id[k] == gCPU[j].topology_id[k]; - } - - if (samePackage) - sCPUToPackage[i] = sCPUToPackage[j]; - } - } - // create package heap and idle package stack sPackageEntries = new(std::nothrow) PackageEntry[packageCount]; if (sPackageEntries == NULL) @@ -1202,7 +1190,7 @@ scheduler_affine_init() if (sPackageUsageHeap == NULL) return B_NO_MEMORY; ObjectDeleter packageHeapDeleter(sPackageUsageHeap); - status_t result = sPackageUsageHeap->GrowHeap(packageCount); + result = sPackageUsageHeap->GrowHeap(packageCount); if (result != B_OK) return B_OK; @@ -1309,7 +1297,5 @@ scheduler_affine_init() packageEntriesDeleter.Detach(); packageHeapDeleter.Detach(); packageListDeleter.Detach(); - cpuToPackageDeleter.Detach(); - cpuToCoreDeleter.Detach(); return B_OK; } From cd8d4e39fd92e1886821dcb21ff08f81f1227317 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 02:17:00 +0200 Subject: [PATCH 055/273] kernel: Introduce scheduler modes of operation --- headers/private/kernel/kscheduler.h | 11 +++ .../kernel/scheduler/scheduler_affine.cpp | 73 ++++++++++++++++++- .../kernel/scheduler/scheduler_simple.cpp | 1 + 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index 2c399b1b50..7746222f38 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -17,6 +17,13 @@ struct scheduling_analysis; struct SchedulerListener; +typedef enum scheduler_mode { + SCHEDULER_MODE_PERFORMANCE, + SCHEDULER_MODE_POWER_SAVING, + // ... + SCHEDULER_MODE_COUNT +} scheduler_mode; + struct scheduler_ops { /*! Enqueues the thread in the ready-to-run queue. The caller must hold the scheduler lock (with disabled interrupts). @@ -68,6 +75,10 @@ struct scheduler_ops { */ void (*start)(void); + /*! Sets scheduler operation mode. + */ + status_t (*set_operation_mode)(scheduler_mode mode); + /*! Dumps scheduler specific thread information. */ void (*dump_thread_data)(Thread* thread); diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 4f4c1be9b8..96aa692d21 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -49,6 +49,11 @@ const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; +static scheduler_mode sSchedulerMode; + +static int32 (*sChooseCore)(void); + + // Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical // processors in the heap belong to. Since there are no cache affinity issues // at this level and the run queue is shared among all logical processors on @@ -523,8 +528,8 @@ affine_update_priority_heaps(int32 cpu, int32 priority) } -static inline int32 -affine_choose_core(void) +static int32 +affine_choose_core_performance(void) { CoreEntry* entry; @@ -546,6 +551,37 @@ affine_choose_core(void) } +static int32 +affine_choose_core_power_saving(void) +{ + CoreEntry* entry; + + // TODO: small tasks packing + if (sPackageUsageHeap->PeekMinimum() != NULL) { + // wake new core + PackageEntry* package = sPackageUsageHeap->PeekMinimum(); + entry = package->fIdleCores.Last(); + } else if (sIdlePackageList->Last() != NULL) { + // wake new package + PackageEntry* package = sIdlePackageList->Last(); + entry = package->fIdleCores.Last(); + } else { + // no idle cores, use least occupied core + entry = sCorePriorityHeap->PeekRoot(); + } + + ASSERT(entry != NULL); + return entry->fCoreID; +} + + +static inline int32 +affine_choose_core(void) +{ + return sChooseCore(); +} + + static inline int32 affine_choose_cpu(int32 core) { @@ -1092,6 +1128,37 @@ affine_start(void) } +static status_t +affine_set_operation_mode(scheduler_mode mode) +{ + if (mode != SCHEDULER_MODE_PERFORMANCE + && mode != SCHEDULER_MODE_POWER_SAVING) { + return B_BAD_VALUE; + } + +#ifdef TRACE_SCHEDULER + const char* modeNames = { "performance", "power saving" }; +#endif + TRACE("switching scheduler to %s mode\n", modeNames[mode]); + + sSchedulerMode = mode; + switch (mode) { + case SCHEDULER_MODE_PERFORMANCE: + sChooseCore = affine_choose_core_performance; + break; + + case SCHEDULER_MODE_POWER_SAVING: + sChooseCore = affine_choose_core_power_saving; + break; + + default: + break; + } + + return B_OK; +} + + static scheduler_ops kAffineOps = { affine_enqueue_in_run_queue, affine_reschedule, @@ -1101,6 +1168,7 @@ static scheduler_ops kAffineOps = { affine_on_thread_init, affine_on_thread_destroy, affine_start, + affine_set_operation_mode, affine_dump_thread_data }; @@ -1278,6 +1346,7 @@ scheduler_affine_init() return result; } + affine_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); gScheduler = &kAffineOps; add_debugger_command_etc("run_queue", &dump_run_queue, diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index 951994e2dd..e02f4d34c2 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -720,6 +720,7 @@ static scheduler_ops kSimpleOps = { simple_on_thread_init, simple_on_thread_destroy, simple_start, + NULL, simple_dump_thread_data }; From 84812e60330406dba1151fe43d1d61ddeb498466 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 02:20:09 +0200 Subject: [PATCH 056/273] scheduler_affine: Correctly assign CPUs to idle threads --- src/system/kernel/scheduler/scheduler_affine.cpp | 5 ++--- src/system/kernel/thread.cpp | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 96aa692d21..9e03cb3531 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -614,9 +614,8 @@ affine_enqueue(Thread* thread, bool newOne) targetCore = sCPUToCore[targetCPU]; ASSERT(targetCore == schedulerThreadData->previous_core); } else if (schedulerThreadData->previous_core < 0) { - if (thread->priority == B_IDLE_PRIORITY) { - static int32 idleThreads = 0; - targetCPU = idleThreads++; + if (thread_is_idle_thread(thread)) { + targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; } else { targetCore = affine_choose_core(); diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 24ed01d800..4c4f142ed3 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2720,6 +2720,7 @@ thread_init(kernel_args *args) } gCPU[i].running_thread = thread; + thread->previous_cpu = &gCPU[i]; thread->team = team_get_kernel_team(); thread->priority = B_IDLE_PRIORITY; From 74192fd984c9dfc06ff31c09675ecd242c842400 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 02:21:14 +0200 Subject: [PATCH 057/273] scheduler_affine: Fix compilation warning --- src/system/kernel/scheduler/scheduler_affine.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 9e03cb3531..5941b62fbf 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -1191,6 +1191,9 @@ traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) case CPU_TOPOLOGY_PACKAGE: packageID = node->id; break; + + default: + break; } for (int32 i = 0; i < node->children_count; i++) From ea79da95009a2f916bebbd5d3b57281035548e25 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 02:30:20 +0200 Subject: [PATCH 058/273] kernel: Remove support for thread_queue --- headers/private/kernel/thread.h | 5 -- headers/private/kernel/thread_types.h | 7 --- src/system/kernel/thread.cpp | 66 +-------------------------- 3 files changed, 2 insertions(+), 76 deletions(-) diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 5ea3f7cbb3..469e339290 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -73,11 +73,6 @@ using BKernel::ThreadCreationAttributes; extern "C" { #endif -void thread_enqueue(Thread *t, struct thread_queue *q); -Thread *thread_lookat_queue(struct thread_queue *q); -Thread *thread_dequeue(struct thread_queue *q); -Thread *thread_dequeue_id(struct thread_queue *q, thread_id id); - void thread_at_kernel_entry(bigtime_t now); // called when the thread enters the kernel on behalf of the thread void thread_at_kernel_exit(void); diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index b182032bd1..9afb2f516d 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -418,7 +418,6 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, int64 serial_number; // immutable after adding thread to hash Thread *hash_next; // protected by thread hash lock Thread *team_next; // protected by team lock and fLock - Thread *queue_next; // protected by scheduler lock timer alarm; // protected by scheduler lock char name[B_OS_NAME_LENGTH]; // protected by fLock int32 priority; // protected by scheduler lock @@ -781,12 +780,6 @@ using BKernel::ProcessGroup; using BKernel::ProcessGroupList; -struct thread_queue { - Thread* head; - Thread* tail; -}; - - #endif // !_ASSEMBLER diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 4c4f142ed3..391a487fbe 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -165,7 +165,6 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) serial_number(-1), hash_next(NULL), team_next(NULL), - queue_next(NULL), priority(-1), io_priority(-1), cpu(cpu), @@ -1698,8 +1697,8 @@ _dump_thread_info(Thread *thread, bool shortInfo) thread->id); kprintf("serial_number: %" B_PRId64 "\n", thread->serial_number); kprintf("name: \"%s\"\n", thread->name); - kprintf("hash_next: %p\nteam_next: %p\nq_next: %p\n", - thread->hash_next, thread->team_next, thread->queue_next); + kprintf("hash_next: %p\nteam_next: %p\n", + thread->hash_next, thread->team_next); kprintf("priority: %" B_PRId32 " (I/O: %" B_PRId32 ")\n", thread->priority, thread->io_priority); kprintf("state: %s\n", state_to_text(thread, thread->state)); @@ -2348,67 +2347,6 @@ thread_reset_for_exec(void) } -/*! Insert a thread to the tail of a queue */ -void -thread_enqueue(Thread *thread, struct thread_queue *queue) -{ - thread->queue_next = NULL; - if (queue->head == NULL) { - queue->head = thread; - queue->tail = thread; - } else { - queue->tail->queue_next = thread; - queue->tail = thread; - } -} - - -Thread * -thread_lookat_queue(struct thread_queue *queue) -{ - return queue->head; -} - - -Thread * -thread_dequeue(struct thread_queue *queue) -{ - Thread *thread = queue->head; - - if (thread != NULL) { - queue->head = thread->queue_next; - if (queue->tail == thread) - queue->tail = NULL; - } - return thread; -} - - -Thread * -thread_dequeue_id(struct thread_queue *q, thread_id id) -{ - Thread *thread; - Thread *last = NULL; - - thread = q->head; - while (thread != NULL) { - if (thread->id == id) { - if (last == NULL) - q->head = thread->queue_next; - else - last->queue_next = thread->queue_next; - - if (q->tail == thread) - q->tail = last; - break; - } - last = thread; - thread = thread->queue_next; - } - return thread; -} - - thread_id allocate_thread_id() { From 7ea42e7addccef196b840aa5e4921adfe13be44d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 02:38:57 +0200 Subject: [PATCH 059/273] kernel: Remove invoke_scheduler_if_idle --- headers/private/kernel/cpu.h | 1 - headers/private/kernel/smp.h | 3 +-- .../kernel/scheduler/scheduler_affine.cpp | 17 ++--------------- .../kernel/scheduler/scheduler_simple.cpp | 17 ++--------------- src/system/kernel/smp.cpp | 10 ---------- 5 files changed, 5 insertions(+), 43 deletions(-) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index d3cf0a8bcd..e833a944cd 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -71,7 +71,6 @@ typedef struct cpu_ent { Thread* running_thread; Thread* previous_thread; bool invoke_scheduler; - bool invoke_scheduler_if_idle; bool disabled; // CPU topology information diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index aa19ad35d9..90427fb825 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -22,8 +22,7 @@ enum { SMP_MSG_GLOBAL_INVALIDATE_PAGES, SMP_MSG_CPU_HALT, SMP_MSG_CALL_FUNCTION, - SMP_MSG_RESCHEDULE, - SMP_MSG_RESCHEDULE_IF_IDLE + SMP_MSG_RESCHEDULE }; enum { diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 5941b62fbf..4360c78e2b 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -656,10 +656,9 @@ affine_enqueue(Thread* thread, bool newOne) // reschedule anyway and update its heap key to the correct value. affine_update_priority_heaps(targetCPU, threadPriority); - if (targetCPU == smp_get_current_cpu()) { + if (targetCPU == smp_get_current_cpu()) gCPU[targetCPU].invoke_scheduler = true; - gCPU[targetCPU].invoke_scheduler_if_idle = false; - } else { + else { smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); } @@ -840,7 +839,6 @@ reschedule_event(timer *unused) thread->scheduler_data->lost_cpu = true; thread->cpu->invoke_scheduler = true; - thread->cpu->invoke_scheduler_if_idle = false; thread->cpu->preempted = 1; return B_HANDLED_INTERRUPT; } @@ -960,17 +958,6 @@ affine_reschedule(void) { Thread* oldThread = thread_get_current_thread(); - // check whether we're only supposed to reschedule, if the current thread - // is idle - if (oldThread->cpu->invoke_scheduler) { - oldThread->cpu->invoke_scheduler = false; - if (oldThread->cpu->invoke_scheduler_if_idle - && oldThread->priority != B_IDLE_PRIORITY) { - oldThread->cpu->invoke_scheduler_if_idle = false; - return; - } - } - int32 thisCPU = smp_get_current_cpu(); int32 thisCore = sCPUToCore[thisCPU]; diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index e02f4d34c2..e4d413b2b1 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -323,10 +323,9 @@ simple_enqueue(Thread* thread, bool newOne) // reschedule anyway and update its heap key to the correct value. sCPUHeap->ModifyKey(&sCPUEntries[targetCPU], threadPriority); - if (targetCPU == smp_get_current_cpu()) { + if (targetCPU == smp_get_current_cpu()) gCPU[targetCPU].invoke_scheduler = true; - gCPU[targetCPU].invoke_scheduler_if_idle = false; - } else { + else { smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); } @@ -428,7 +427,6 @@ reschedule_event(timer* /* unused */) thread->scheduler_data->lost_cpu = true; thread->cpu->invoke_scheduler = true; - thread->cpu->invoke_scheduler_if_idle = false; thread->cpu->preempted = 1; return B_HANDLED_INTERRUPT; } @@ -547,17 +545,6 @@ simple_reschedule(void) { Thread* oldThread = thread_get_current_thread(); - // check whether we're only supposed to reschedule, if the current thread - // is idle - if (oldThread->cpu->invoke_scheduler) { - oldThread->cpu->invoke_scheduler = false; - if (oldThread->cpu->invoke_scheduler_if_idle - && oldThread->priority != B_IDLE_PRIORITY) { - oldThread->cpu->invoke_scheduler_if_idle = false; - return; - } - } - int32 thisCPU = smp_get_current_cpu(); TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index bf5f7c7603..e28faf619c 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -740,16 +740,6 @@ process_pending_ici(int32 currentCPU) { cpu_ent* cpu = thread_get_current_thread()->cpu; cpu->invoke_scheduler = true; - cpu->invoke_scheduler_if_idle = false; - break; - } - case SMP_MSG_RESCHEDULE_IF_IDLE: - { - cpu_ent* cpu = thread_get_current_thread()->cpu; - if (!cpu->invoke_scheduler) { - cpu->invoke_scheduler = true; - cpu->invoke_scheduler_if_idle = true; - } break; } From afe1735d7d9bd76a8a382c4e5f6271b3afcdaaf1 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 19:19:28 +0200 Subject: [PATCH 060/273] scheduler_affine: Expire old cache affinities Performance mode: If there have been a lot of activity on the core since the thread went sleep its data in cache probably has been overwritten. Power saving mode: If the thread went to sleep a long time ago either there has been a lot of activity on its core or the core has been idle and it may be more efficient to wake another one. --- .../kernel/scheduler/scheduler_affine.cpp | 86 ++++++++++++++++--- 1 file changed, 73 insertions(+), 13 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 4360c78e2b..094dc64a28 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -48,6 +48,7 @@ const bigtime_t kThreadQuantum = 1000; const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; +const bigtime_t kCacheExpire = 100000; static scheduler_mode sSchedulerMode; @@ -69,6 +70,8 @@ static AffineCPUHeap* sCPUPriorityHeaps; struct CoreEntry : public HeapLinkImpl, DoublyLinkedListLinkImpl { int32 fCoreID; + + bigtime_t fActiveTime; }; static CoreEntry* sCoreEntries; @@ -131,6 +134,7 @@ struct scheduler_thread_data { bigtime_t quantum_start; bigtime_t went_sleep; + bigtime_t went_sleep_active; int32 previous_core; }; @@ -146,6 +150,7 @@ scheduler_thread_data::Init() stolen_time = 0; went_sleep = 0; + went_sleep_active = 0; lost_cpu = false; cpu_bound = true; @@ -378,6 +383,31 @@ dump_idle_cores(int argc, char** argv) } +static inline bool +affine_has_cache_expired(Thread* thread) +{ + if (thread_is_idle_thread(thread)) + return false; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; + switch (sSchedulerMode) { + case SCHEDULER_MODE_PERFORMANCE: + return coreEntry->fActiveTime + - schedulerThreadData->went_sleep_active > kCacheExpire; + + case SCHEDULER_MODE_POWER_SAVING: + return system_time() - schedulerThreadData->went_sleep + > kCacheExpire; + + default: + return true; + } +} + + static void affine_dump_thread_data(Thread* thread) { @@ -396,8 +426,16 @@ affine_dump_thread_data(Thread* thread) additionalPenalty, schedulerThreadData->additional_penalty); kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", schedulerThreadData->stolen_time); + kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", + schedulerThreadData->went_sleep); + kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", + schedulerThreadData->went_sleep_active); kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", schedulerThreadData->previous_core); + if (schedulerThreadData->previous_core > 0 + && affine_has_cache_expired(thread)) { + kprintf("\tcache affinity has expired\n"); + } } @@ -613,7 +651,9 @@ affine_enqueue(Thread* thread, bool newOne) targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; ASSERT(targetCore == schedulerThreadData->previous_core); - } else if (schedulerThreadData->previous_core < 0) { + } else if (schedulerThreadData->previous_core < 0 + || (newOne && affine_has_cache_expired(thread))) { + if (thread_is_idle_thread(thread)) { targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; @@ -950,6 +990,34 @@ affine_dequeue_thread(int32 thisCPU) } +static inline void +affine_track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) +{ + if (!thread_is_idle_thread(oldThread)) { + bigtime_t active + = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) + + (oldThread->user_time - oldThread->cpu->last_user_time); + + oldThread->cpu->active_time += active; + sCoreEntries[thisCore].fActiveTime += active; + } + + if (!thread_is_idle_thread(nextThread)) { + oldThread->cpu->last_kernel_time = nextThread->kernel_time; + oldThread->cpu->last_user_time = nextThread->user_time; + } +} + + +static inline void +affine_thread_goes_sleep(Thread* thread, int32 thisCore) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + schedulerThreadData->went_sleep = system_time(); + schedulerThreadData->went_sleep_active = sCoreEntries[thisCore].fActiveTime; +} + + /*! Runs the scheduler. Note: expects thread spinlock to be held */ @@ -996,13 +1064,13 @@ affine_reschedule(void) break; case B_THREAD_SUSPENDED: - schedulerOldThreadData->went_sleep = system_time(); + affine_thread_goes_sleep(oldThread, thisCore); TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: break; default: - schedulerOldThreadData->went_sleep = system_time(); + affine_thread_goes_sleep(oldThread, thisCore); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, oldThread->next_state); break; @@ -1049,16 +1117,7 @@ affine_reschedule(void) scheduler_update_thread_times(oldThread, nextThread); // track CPU activity - if (!thread_is_idle_thread(oldThread)) { - oldThread->cpu->active_time += - (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time); - } - - if (!thread_is_idle_thread(nextThread)) { - oldThread->cpu->last_kernel_time = nextThread->kernel_time; - oldThread->cpu->last_user_time = nextThread->user_time; - } + affine_track_cpu_activity(oldThread, nextThread, thisCore); if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; @@ -1282,6 +1341,7 @@ scheduler_affine_init() for (int32 i = 0; i < coreCount; i++) { sCoreEntries[i].fCoreID = i; + sCoreEntries[i].fActiveTime = 0; status_t result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); if (result != B_OK) From 5cf9b69b498bff8d197b1ea6ca2b2c47094890fd Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 21:24:05 +0200 Subject: [PATCH 061/273] kernel/util: Minor improvements in Heap and MinMaxHeap * [MinMax]Heap::ModifyKey(): Do not attempt to move node if the key actually hasn't changed. * Allow allocating initial array at construction. --- headers/private/kernel/util/Heap.h | 21 +++++++++++++++---- headers/private/kernel/util/MinMaxHeap.h | 26 ++++++++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h index 653e4b09e8..5d0f896ab3 100644 --- a/headers/private/kernel/util/Heap.h +++ b/headers/private/kernel/util/Heap.h @@ -75,6 +75,7 @@ template* link); void _MoveDown(HeapLink* link); @@ -167,6 +168,17 @@ HEAP_CLASS_NAME::Heap() } +HEAP_TEMPLATE_LIST +HEAP_CLASS_NAME::Heap(int initialSize) + : + fElements(NULL), + fLastElement(0), + fSize(0) +{ + _GrowHeap(initialSize); +} + + HEAP_TEMPLATE_LIST HEAP_CLASS_NAME::~Heap() { @@ -204,7 +216,7 @@ HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey) if (sCompare(newKey, oldKey)) _MoveUp(link); - else + else if (sCompare(oldKey, newKey)) _MoveDown(link); } @@ -256,9 +268,10 @@ HEAP_CLASS_NAME::Insert(Element* element, Key key) HEAP_TEMPLATE_LIST status_t -HEAP_CLASS_NAME::_GrowHeap() +HEAP_CLASS_NAME::_GrowHeap(int minimalSize) { - int newSize = max_c(fSize * 2, 4); + minimalSize = minimalSize % 2 ? minimalSize : minimalSize + 1; + int newSize = max_c(max_c(fSize * 2, 4), minimalSize); size_t arraySize = newSize * sizeof(Element*); Element** newBuffer diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 01a48d1be9..0e9f40bb2d 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -70,6 +70,7 @@ template* link); void _MoveDown(MinMaxHeapLink* link); bool _ChangeTree(MinMaxHeapLink* link); @@ -166,6 +168,19 @@ MIN_MAX_HEAP_CLASS_NAME::MinMaxHeap() } +MIN_MAX_HEAP_TEMPLATE_LIST +MIN_MAX_HEAP_CLASS_NAME::MinMaxHeap(int initialSize) + : + fMinElements(NULL), + fMinLastElement(0), + fMaxElements(NULL), + fMaxLastElement(0), + fSize(0) +{ + _GrowHeap(initialSize); +} + + MIN_MAX_HEAP_TEMPLATE_LIST MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap() { @@ -220,7 +235,10 @@ MIN_MAX_HEAP_CLASS_NAME::ModifyKey(Element* element, Key newKey) Key oldKey = link->fKey; link->fKey = newKey; - if (sCompare(newKey, oldKey) && link->fMinTree) + if (!sCompare(newKey, oldKey) && !sCompare(oldKey, newKey)) + return; + + if (sCompare(newKey, oldKey) ^ !link->fMinTree) _MoveUp(link); else _MoveDown(link); @@ -273,7 +291,7 @@ MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key) { if (min_c(fMinLastElement, fMaxLastElement) == fSize) { ASSERT(max_c(fMinLastElement, fMaxLastElement) == fSize); - status_t result = GrowHeap(); + status_t result = _GrowHeap(); if (result != B_OK) return result; } @@ -300,7 +318,7 @@ MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key) MIN_MAX_HEAP_TEMPLATE_LIST status_t -MIN_MAX_HEAP_CLASS_NAME::GrowHeap(int minimalSize) +MIN_MAX_HEAP_CLASS_NAME::_GrowHeap(int minimalSize) { minimalSize = minimalSize % 2 ? minimalSize : minimalSize + 1; int newSize = max_c(max_c(fSize * 4, 4), minimalSize); From 7aba623f524cb190460597f5d54bd9e890363def Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 21 Oct 2013 21:39:40 +0200 Subject: [PATCH 062/273] scheduler_affine: Balance number of threads assigned to CPUs When the thread cannot be run immediately assign it to the core with lowest number of CPU bound threads and assigned threads. --- .../kernel/scheduler/scheduler_affine.cpp | 178 +++++++++++++----- 1 file changed, 134 insertions(+), 44 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 094dc64a28..18d3139fab 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -52,7 +52,7 @@ const bigtime_t kCacheExpire = 100000; static scheduler_mode sSchedulerMode; -static int32 (*sChooseCore)(void); +static int32 (*sChooseCore)(int32 priority); // Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical @@ -67,16 +67,28 @@ typedef MinMaxHeap AffineCPUHeap; static CPUEntry* sCPUEntries; static AffineCPUHeap* sCPUPriorityHeaps; -struct CoreEntry : public HeapLinkImpl, - DoublyLinkedListLinkImpl { +struct CoreEntry : public DoublyLinkedListLinkImpl { + HeapLink fPriorityHeapLink; + HeapLink fThreadHeapLink; + int32 fCoreID; bigtime_t fActiveTime; + + int32 fCPUBoundThreads; + int32 fThreads; }; static CoreEntry* sCoreEntries; -typedef Heap AffineCoreHeap; -static AffineCoreHeap* sCorePriorityHeap; +typedef Heap, + HeapMemberGetLink > + AffineCorePriorityHeap; +static AffineCorePriorityHeap* sCorePriorityHeap; + +typedef Heap, + HeapMemberGetLink > + AffineCoreThreadHeap; +static AffineCoreThreadHeap* sCoreThreadHeap; // sPackageUsageHeap is used to decide which core should be woken up from the // idle state. When aiming for performance we should use as many packages as @@ -249,7 +261,7 @@ dump_run_queue(int argc, char **argv) static void dump_heap(AffineCPUHeap* heap) { - AffineCPUHeap temp; + AffineCPUHeap temp(smp_get_num_cpus()); kprintf("cpu priority actual priority\n"); CPUEntry* entry = heap->PeekMinimum(); @@ -278,13 +290,13 @@ dump_heap(AffineCPUHeap* heap) static int dump_cpu_heap(int argc, char** argv) { - AffineCoreHeap temp; + AffineCorePriorityHeap temp(sRunQueueCount); kprintf("core priority\n"); CoreEntry* entry = sCorePriorityHeap->PeekRoot(); while (entry) { int32 core = entry->fCoreID; - int32 key = AffineCoreHeap::GetKey(entry); + int32 key = AffineCorePriorityHeap::GetKey(entry); kprintf("%4" B_PRId32 " %8" B_PRId32 "\n", core, key); sCorePriorityHeap->RemoveRoot(); @@ -295,12 +307,35 @@ dump_cpu_heap(int argc, char** argv) entry = temp.PeekRoot(); while (entry) { - int32 key = AffineCoreHeap::GetKey(entry); + int32 key = AffineCorePriorityHeap::GetKey(entry); temp.RemoveRoot(); sCorePriorityHeap->Insert(entry, key); entry = temp.PeekRoot(); } + AffineCoreThreadHeap temp2(sRunQueueCount); + + kprintf("\ncore key threads cpu-bound\n"); + entry = sCoreThreadHeap->PeekRoot(); + while (entry) { + int32 key = AffineCoreThreadHeap::GetKey(entry); + kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 "\n", + entry->fCoreID, key, entry->fThreads, entry->fCPUBoundThreads); + + sCoreThreadHeap->RemoveRoot(); + temp2.Insert(entry, key); + + entry = sCoreThreadHeap->PeekRoot(); + } + + entry = temp2.PeekRoot(); + while (entry) { + int32 key = AffineCoreThreadHeap::GetKey(entry); + temp2.RemoveRoot(); + sCoreThreadHeap->Insert(entry, key); + entry = temp2.PeekRoot(); + } + for (int32 i = 0; i < sRunQueueCount; i++) { kprintf("\nCore %" B_PRId32 " heap:\n", i); dump_heap(&sCPUPriorityHeaps[i]); @@ -339,8 +374,7 @@ dump_idle_cores(int argc, char** argv) } else kprintf("No idle packages.\n"); - AffinePackageHeap temp; - temp.GrowHeap(smp_get_num_cpus()); + AffinePackageHeap temp(smp_get_num_cpus()); kprintf("\nPackages with idle cores:\n"); PackageEntry* entry = sPackageUsageHeap->PeekMinimum(); @@ -439,6 +473,22 @@ affine_dump_thread_data(Thread* thread) } +static void +affine_update_thread_heaps(int32 core) +{ + CoreEntry* entry = &sCoreEntries[core]; + + ASSERT(entry->fCPUBoundThreads >= 0 + && entry->fCPUBoundThreads <= entry->fThreads); + ASSERT(entry->fThreads >= 0 + && entry->fThreads <= thread_max_threads()); + + int32 newKey = entry->fCPUBoundThreads * thread_max_threads(); + newKey += entry->fThreads; + sCoreThreadHeap->ModifyKey(entry, newKey); +} + + static inline void affine_increase_penalty(Thread* thread) { @@ -455,6 +505,13 @@ affine_increase_penalty(Thread* thread) ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); const int kMinimalPriority = affine_get_minimal_priority(thread); if (thread->priority - oldPenalty <= kMinimalPriority) { + int32 core = schedulerThreadData->previous_core; + ASSERT(core >= 0); + if (schedulerThreadData->additional_penalty == 0) { + sCoreEntries[core].fCPUBoundThreads++; + affine_update_thread_heaps(core); + } + schedulerThreadData->priority_penalty = oldPenalty; schedulerThreadData->additional_penalty++; } @@ -468,31 +525,20 @@ affine_cancel_penalty(Thread* thread) if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); + + if (schedulerThreadData->additional_penalty != 0) { + int32 core = schedulerThreadData->previous_core; + ASSERT(core >= 0); + + sCoreEntries[core].fCPUBoundThreads--; + affine_update_thread_heaps(core); + } + schedulerThreadData->priority_penalty = 0; schedulerThreadData->additional_penalty = 0; } -/*! Returns the most idle CPU based on the active time counters. - Note: thread lock must be held when entering this function -*/ -#if 0 -static int32 -affine_get_most_idle_cpu() -{ - int32 targetCPU = -1; - for (int32 i = 0; i < smp_get_num_cpus(); i++) { - if (gCPU[i].disabled) - continue; - if (targetCPU < 0 || sRunQueueSize[i] < sRunQueueSize[targetCPU]) - targetCPU = i; - } - - return targetCPU; -} -#endif - - static inline void affine_update_priority_heaps(int32 cpu, int32 priority) { @@ -502,7 +548,7 @@ affine_update_priority_heaps(int32 cpu, int32 priority) int32 maxPriority = AffineCPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); - int32 corePriority = AffineCoreHeap::GetKey(&sCoreEntries[core]); + int32 corePriority = AffineCorePriorityHeap::GetKey(&sCoreEntries[core]); if (corePriority != maxPriority) { sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); @@ -567,7 +613,7 @@ affine_update_priority_heaps(int32 cpu, int32 priority) static int32 -affine_choose_core_performance(void) +affine_choose_core_performance(int32 priority) { CoreEntry* entry; @@ -582,6 +628,8 @@ affine_choose_core_performance(void) } else { // no idle cores, use least occupied core entry = sCorePriorityHeap->PeekRoot(); + if (AffineCorePriorityHeap::GetKey(entry) >= priority) + entry = sCoreThreadHeap->PeekRoot(); } ASSERT(entry != NULL); @@ -590,7 +638,7 @@ affine_choose_core_performance(void) static int32 -affine_choose_core_power_saving(void) +affine_choose_core_power_saving(int32 priority) { CoreEntry* entry; @@ -606,6 +654,8 @@ affine_choose_core_power_saving(void) } else { // no idle cores, use least occupied core entry = sCorePriorityHeap->PeekRoot(); + if (AffineCorePriorityHeap::GetKey(entry) >= priority) + entry = sCoreThreadHeap->PeekRoot(); } ASSERT(entry != NULL); @@ -614,9 +664,9 @@ affine_choose_core_power_saving(void) static inline int32 -affine_choose_core(void) +affine_choose_core(int32 priority) { - return sChooseCore(); + return sChooseCore(priority); } @@ -629,6 +679,33 @@ affine_choose_cpu(int32 core) } +static void +affine_assign_thread_to_core(Thread* thread, int32 targetCore) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + int32 oldCore = schedulerThreadData->previous_core; + if (oldCore == targetCore) + return; + + if (oldCore >= 0) { + sCoreEntries[oldCore].fThreads--; + if (schedulerThreadData->additional_penalty != 0) + sCoreEntries[oldCore].fCPUBoundThreads--; + + affine_update_thread_heaps(oldCore); + } + + schedulerThreadData->previous_core = targetCore; + if (targetCore >= 0) { + sCoreEntries[targetCore].fThreads++; + if (schedulerThreadData->additional_penalty != 0) + sCoreEntries[targetCore].fCPUBoundThreads++; + affine_update_thread_heaps(targetCore); + } +} + + static void affine_enqueue(Thread* thread, bool newOne) { @@ -658,10 +735,11 @@ affine_enqueue(Thread* thread, bool newOne) targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; } else { - targetCore = affine_choose_core(); + targetCore = affine_choose_core(threadPriority); targetCPU = affine_choose_cpu(targetCore); } - schedulerThreadData->previous_core = targetCore; + + affine_assign_thread_to_core(thread, targetCore); } else { targetCore = schedulerThreadData->previous_core; targetCPU = affine_choose_cpu(targetCore); @@ -1068,6 +1146,7 @@ affine_reschedule(void) TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: + affine_assign_thread_to_core(oldThread, -1); break; default: affine_thread_goes_sleep(oldThread, thisCore); @@ -1302,13 +1381,10 @@ scheduler_affine_init() return B_NO_MEMORY; ArrayDeleter packageEntriesDeleter(sPackageEntries); - sPackageUsageHeap = new(std::nothrow) AffinePackageHeap; + sPackageUsageHeap = new(std::nothrow) AffinePackageHeap(packageCount); if (sPackageUsageHeap == NULL) return B_NO_MEMORY; ObjectDeleter packageHeapDeleter(sPackageUsageHeap); - result = sPackageUsageHeap->GrowHeap(packageCount); - if (result != B_OK) - return B_OK; sIdlePackageList = new(std::nothrow) AffineIdlePackageList; if (sIdlePackageList == NULL) @@ -1334,18 +1410,31 @@ scheduler_affine_init() ArrayDeleter coreEntriesDeleter( sCoreEntries); - sCorePriorityHeap = new AffineCoreHeap; + sCorePriorityHeap = new AffineCorePriorityHeap; if (sCorePriorityHeap == NULL) return B_NO_MEMORY; - ObjectDeleter corePriorityHeapDeleter(sCorePriorityHeap); + ObjectDeleter corePriorityHeapDeleter( + sCorePriorityHeap); + + sCoreThreadHeap = new AffineCoreThreadHeap; + if (sCoreThreadHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter coreThreadHeapDeleter(sCoreThreadHeap); for (int32 i = 0; i < coreCount; i++) { sCoreEntries[i].fCoreID = i; sCoreEntries[i].fActiveTime = 0; + sCoreEntries[i].fThreads = 0; + sCoreEntries[i].fCPUBoundThreads = 0; + status_t result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; + + result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); + if (result != B_OK) + return result; } sCPUPriorityHeaps = new AffineCPUHeap[coreCount]; @@ -1408,6 +1497,7 @@ scheduler_affine_init() runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); + coreThreadHeapDeleter.Detach(); corePriorityHeapDeleter.Detach(); cpuPriorityHeapDeleter.Detach(); coreEntriesDeleter.Detach(); From 2e0ee59462130209dc9bfd49ad05196687551cd8 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 22 Oct 2013 01:11:41 +0200 Subject: [PATCH 063/273] scheduler_affine: Migrate threads from overloaded cores * Keep number of CPU bound threads on cores balanced. * If possible migrate normal threads from cores with cpu bound ones to the less busy cores. --- .../kernel/scheduler/scheduler_affine.cpp | 163 +++++++++++------- 1 file changed, 104 insertions(+), 59 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 18d3139fab..8efd3aa71f 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -89,6 +89,8 @@ typedef Heap, HeapMemberGetLink > AffineCoreThreadHeap; static AffineCoreThreadHeap* sCoreThreadHeap; +static int32 sCPUBoundThreads; +static int32 sAssignedThreads; // sPackageUsageHeap is used to decide which core should be woken up from the // idle state. When aiming for performance we should use as many packages as @@ -508,6 +510,7 @@ affine_increase_penalty(Thread* thread) int32 core = schedulerThreadData->previous_core; ASSERT(core >= 0); if (schedulerThreadData->additional_penalty == 0) { + sCPUBoundThreads++; sCoreEntries[core].fCPUBoundThreads++; affine_update_thread_heaps(core); } @@ -526,14 +529,6 @@ affine_cancel_penalty(Thread* thread) if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); - if (schedulerThreadData->additional_penalty != 0) { - int32 core = schedulerThreadData->previous_core; - ASSERT(core >= 0); - - sCoreEntries[core].fCPUBoundThreads--; - affine_update_thread_heaps(core); - } - schedulerThreadData->priority_penalty = 0; schedulerThreadData->additional_penalty = 0; } @@ -679,30 +674,93 @@ affine_choose_cpu(int32 core) } -static void -affine_assign_thread_to_core(Thread* thread, int32 targetCore) +static bool +affine_should_rebalance(Thread* thread) { - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + if (thread_is_idle_thread(thread)) + return false; - int32 oldCore = schedulerThreadData->previous_core; - if (oldCore == targetCore) + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; + + // If this is a cpu bound thread and we have significantly more such threads + // than the average get rid of this one. + if (schedulerThreadData->additional_penalty != 0) { + int32 averageCPUBound = sCPUBoundThreads / sRunQueueCount; + if (coreEntry->fCPUBoundThreads - averageCPUBound > 1) + return true; + return false; + } + + // If this thread is not cpu bound but we have at least one consider giving + // this one to someone less busy. + int32 averageThread = sAssignedThreads / sRunQueueCount; + if (coreEntry->fCPUBoundThreads > 0) { + CoreEntry* other = sCoreThreadHeap->PeekRoot(); + if (AffineCoreThreadHeap::GetKey(other) <= averageThread) + return true; + } + + // No cpu bound threads - the situation is quite good. Make sure it + // won't get much worse... + const int32 kBalanceThreshold = 3; + return coreEntry->fThreads - averageThread > kBalanceThreshold; +} + + +static void +affine_assign_active_thread_to_core(Thread* thread) +{ + if (thread_is_idle_thread(thread)) return; - if (oldCore >= 0) { - sCoreEntries[oldCore].fThreads--; - if (schedulerThreadData->additional_penalty != 0) - sCoreEntries[oldCore].fCPUBoundThreads--; + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - affine_update_thread_heaps(oldCore); + ASSERT(schedulerThreadData->previous_core >= 0); + int32 core = schedulerThreadData->previous_core; + + sCoreEntries[core].fThreads++; + sAssignedThreads++; + + if (schedulerThreadData->additional_penalty != 0) { + sCoreEntries[core].fCPUBoundThreads++; + sCPUBoundThreads++; } - schedulerThreadData->previous_core = targetCore; - if (targetCore >= 0) { - sCoreEntries[targetCore].fThreads++; - if (schedulerThreadData->additional_penalty != 0) - sCoreEntries[targetCore].fCPUBoundThreads++; - affine_update_thread_heaps(targetCore); + affine_update_thread_heaps(core); +} + + +static inline void +affine_thread_goes_away(Thread* thread) +{ + if (thread_is_idle_thread(thread)) + return; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + ASSERT(schedulerThreadData->previous_core >= 0); + int32 core = schedulerThreadData->previous_core; + + ASSERT(sCoreEntries[core].fThreads > 0); + ASSERT(sCoreEntries[core].fThreads > sCoreEntries[core].fCPUBoundThreads + || (sCoreEntries[core].fThreads == sCoreEntries[core].fCPUBoundThreads + && schedulerThreadData->additional_penalty != 0)); + sCoreEntries[core].fThreads--; + sAssignedThreads--; + + if (schedulerThreadData->additional_penalty != 0) { + ASSERT(sCoreEntries[core].fCPUBoundThreads > 0); + sCoreEntries[core].fCPUBoundThreads--; + sCPUBoundThreads--; } + + affine_update_thread_heaps(core); + + schedulerThreadData->went_sleep = system_time(); + schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; } @@ -728,21 +786,31 @@ affine_enqueue(Thread* thread, bool newOne) targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; ASSERT(targetCore == schedulerThreadData->previous_core); + + if (newOne) + affine_assign_active_thread_to_core(thread); } else if (schedulerThreadData->previous_core < 0 - || (newOne && affine_has_cache_expired(thread))) { + || (newOne && affine_has_cache_expired(thread)) + || affine_should_rebalance(thread)) { if (thread_is_idle_thread(thread)) { targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; } else { + if (!newOne) + affine_thread_goes_away(thread); + targetCore = affine_choose_core(threadPriority); targetCPU = affine_choose_cpu(targetCore); } - affine_assign_thread_to_core(thread, targetCore); + schedulerThreadData->previous_core = targetCore; + affine_assign_active_thread_to_core(thread); } else { targetCore = schedulerThreadData->previous_core; targetCPU = affine_choose_cpu(targetCore); + if (newOne) + affine_assign_active_thread_to_core(thread); } TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, @@ -815,26 +883,6 @@ affine_put_back(Thread* thread) #if 0 -/*! Dequeues the thread after the given \a prevThread from the run queue. -*/ -static inline Thread * -dequeue_from_run_queue(Thread *prevThread, int32 currentCPU) -{ - Thread *resultThread = NULL; - if (prevThread != NULL) { - resultThread = prevThread->queue_next; - prevThread->queue_next = resultThread->queue_next; - } else { - resultThread = sRunQueue[currentCPU]; - sRunQueue[currentCPU] = resultThread->queue_next; - } - sRunQueueSize[currentCPU]--; - resultThread->scheduler_data->fLastQueue = -1; - - return resultThread; -} - - /*! Looks for a possible thread to grab/run from another CPU. Note: thread lock must be held when entering this function */ @@ -899,12 +947,17 @@ affine_set_thread_priority(Thread *thread, int32 priority) thread->id, priority, thread->priority, affine_get_effective_priority(thread)); - if (thread->state == B_THREAD_RUNNING) + if (thread->state == B_THREAD_RUNNING) { + affine_thread_goes_away(thread); affine_update_priority_heaps(thread->cpu->cpu_num, priority); + } if (thread->state != B_THREAD_READY) { affine_cancel_penalty(thread); thread->priority = priority; + + if (thread->state == B_THREAD_RUNNING) + affine_assign_active_thread_to_core(thread); return; } @@ -921,6 +974,7 @@ affine_set_thread_priority(Thread *thread, int32 priority) int32 previousCore = thread->scheduler_data->previous_core; ASSERT(previousCore >= 0); sRunQueues[previousCore].Remove(thread); + affine_thread_goes_away(thread); // set priority and re-insert affine_cancel_penalty(thread); @@ -1087,15 +1141,6 @@ affine_track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) } -static inline void -affine_thread_goes_sleep(Thread* thread, int32 thisCore) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - schedulerThreadData->went_sleep = system_time(); - schedulerThreadData->went_sleep_active = sCoreEntries[thisCore].fActiveTime; -} - - /*! Runs the scheduler. Note: expects thread spinlock to be held */ @@ -1142,14 +1187,14 @@ affine_reschedule(void) break; case B_THREAD_SUSPENDED: - affine_thread_goes_sleep(oldThread, thisCore); + affine_thread_goes_away(oldThread); TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: - affine_assign_thread_to_core(oldThread, -1); + affine_thread_goes_away(oldThread); break; default: - affine_thread_goes_sleep(oldThread, thisCore); + affine_thread_goes_away(oldThread); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, oldThread->next_state); break; From f823aacf59cceb227850475367a6c492eef08667 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 22 Oct 2013 01:21:51 +0200 Subject: [PATCH 064/273] scheduler_affine: Remove old code --- .../kernel/scheduler/scheduler_affine.cpp | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 8efd3aa71f..c4f4671bd8 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -882,58 +882,6 @@ affine_put_back(Thread* thread) } -#if 0 -/*! Looks for a possible thread to grab/run from another CPU. - Note: thread lock must be held when entering this function -*/ -static Thread * -steal_thread_from_other_cpus(int32 currentCPU) -{ - // look through the active CPUs - find the one - // that has a) threads available to steal, and - // b) out of those, the one that's the most CPU-bound - // TODO: make this more intelligent along with enqueue - // - we need to try and maintain a reasonable balance - // in run queue sizes across CPUs, and also try to maintain - // an even distribution of cpu bound / interactive threads - int32 targetCPU = -1; - for (int32 i = 0; i < smp_get_num_cpus(); i++) { - // skip CPUs that have either no or only one thread - if (i == currentCPU || sRunQueueSize[i] < 2) - continue; - - // out of the CPUs with threads available to steal, - // pick whichever one is generally the most CPU bound. - if (targetCPU < 0 - || sRunQueue[i]->priority > sRunQueue[targetCPU]->priority - || (sRunQueue[i]->priority == sRunQueue[targetCPU]->priority - && sRunQueueSize[i] > sRunQueueSize[targetCPU])) - targetCPU = i; - } - - if (targetCPU < 0) - return NULL; - - Thread* nextThread = sRunQueue[targetCPU]; - Thread* prevThread = NULL; - - while (nextThread != NULL) { - // grab the highest priority non-pinned thread - // out of this CPU's queue, dequeue and return it - if (nextThread->pinned_to_cpu <= 0) { - dequeue_from_run_queue(prevThread, targetCPU); - return nextThread; - } - - prevThread = nextThread; - nextThread = nextThread->queue_next; - } - - return NULL; -} -#endif - - /*! Sets the priority of a thread. Note: thread lock must be held when entering this function */ From 8d471bc3d9d3c67df8d18b8212b47b488a25bec0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 22 Oct 2013 23:52:40 +0200 Subject: [PATCH 065/273] scheduler_affine: Store cores with CPU bound threads separately This is preparation for small task packing. We want to have as many idle cores as possible. To achieve that we put all threads on the most heavily loaded core (so the other ones can become idle). However, we don't really want to do that if there are CPU bound tasks and if any of the cores becomes overloaded. --- .../kernel/scheduler/scheduler_affine.cpp | 163 +++++++++++++----- 1 file changed, 119 insertions(+), 44 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index c4f4671bd8..c4b2f67e27 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -50,6 +50,8 @@ const bigtime_t kMaxThreadQuantum = 10000; const bigtime_t kCacheExpire = 100000; +static bigtime_t sDisableSmallTaskPacking; + static scheduler_mode sSchedulerMode; static int32 (*sChooseCore)(int32 priority); @@ -69,7 +71,7 @@ static AffineCPUHeap* sCPUPriorityHeaps; struct CoreEntry : public DoublyLinkedListLinkImpl { HeapLink fPriorityHeapLink; - HeapLink fThreadHeapLink; + MinMaxHeapLink fThreadHeapLink; int32 fCoreID; @@ -85,10 +87,12 @@ typedef Heap, AffineCorePriorityHeap; static AffineCorePriorityHeap* sCorePriorityHeap; -typedef Heap, - HeapMemberGetLink > +typedef MinMaxHeap, + MinMaxHeapMemberGetLink > AffineCoreThreadHeap; static AffineCoreThreadHeap* sCoreThreadHeap; +static AffineCoreThreadHeap* sCoreCPUBoundThreadHeap; + static int32 sCPUBoundThreads; static int32 sAssignedThreads; @@ -289,13 +293,44 @@ dump_heap(AffineCPUHeap* heap) } +static void +dump_core_thread_heap(AffineCoreThreadHeap* heap) +{ + AffineCoreThreadHeap temp(sRunQueueCount); + + CoreEntry* entry = heap->PeekMinimum(); + while (entry) { + int32 key = AffineCoreThreadHeap::GetKey(entry); + kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 "\n", + entry->fCoreID, key, entry->fThreads, entry->fCPUBoundThreads); + + heap->RemoveMinimum(); + temp.Insert(entry, key); + + entry = heap->PeekMinimum(); + } + + entry = temp.PeekMinimum(); + while (entry) { + int32 key = AffineCoreThreadHeap::GetKey(entry); + temp.RemoveMinimum(); + heap->Insert(entry, key); + entry = temp.PeekMinimum(); + } +} + + static int dump_cpu_heap(int argc, char** argv) { AffineCorePriorityHeap temp(sRunQueueCount); - kprintf("core priority\n"); CoreEntry* entry = sCorePriorityHeap->PeekRoot(); + if (entry != NULL) + kprintf("core priority\n"); + else + kprintf("No active cores.\n"); + while (entry) { int32 core = entry->fCoreID; int32 key = AffineCorePriorityHeap::GetKey(entry); @@ -315,28 +350,9 @@ dump_cpu_heap(int argc, char** argv) entry = temp.PeekRoot(); } - AffineCoreThreadHeap temp2(sRunQueueCount); - kprintf("\ncore key threads cpu-bound\n"); - entry = sCoreThreadHeap->PeekRoot(); - while (entry) { - int32 key = AffineCoreThreadHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 "\n", - entry->fCoreID, key, entry->fThreads, entry->fCPUBoundThreads); - - sCoreThreadHeap->RemoveRoot(); - temp2.Insert(entry, key); - - entry = sCoreThreadHeap->PeekRoot(); - } - - entry = temp2.PeekRoot(); - while (entry) { - int32 key = AffineCoreThreadHeap::GetKey(entry); - temp2.RemoveRoot(); - sCoreThreadHeap->Insert(entry, key); - entry = temp2.PeekRoot(); - } + dump_core_thread_heap(sCoreThreadHeap); + dump_core_thread_heap(sCoreCPUBoundThreadHeap); for (int32 i = 0; i < sRunQueueCount; i++) { kprintf("\nCore %" B_PRId32 " heap:\n", i); @@ -487,7 +503,32 @@ affine_update_thread_heaps(int32 core) int32 newKey = entry->fCPUBoundThreads * thread_max_threads(); newKey += entry->fThreads; - sCoreThreadHeap->ModifyKey(entry, newKey); + + int32 oldKey = AffineCoreThreadHeap::GetKey(entry); + + if (oldKey == newKey) + return; + + if (newKey > thread_max_threads()) { + if (oldKey <= thread_max_threads()) { + sCoreThreadHeap->ModifyKey(entry, -1); + ASSERT(sCoreThreadHeap->PeekMinimum() == entry); + sCoreThreadHeap->RemoveMinimum(); + ASSERT(sCoreThreadHeap->PeekMinimum() != entry); + + sCoreCPUBoundThreadHeap->Insert(entry, newKey); + } else + sCoreCPUBoundThreadHeap->ModifyKey(entry, newKey); + } else { + if (oldKey > thread_max_threads()) { + sCoreCPUBoundThreadHeap->ModifyKey(entry, -1); + ASSERT(sCoreCPUBoundThreadHeap->PeekMinimum() == entry); + sCoreCPUBoundThreadHeap->RemoveMinimum(); + + sCoreThreadHeap->Insert(entry, newKey); + } else + sCoreThreadHeap->ModifyKey(entry, newKey); + } } @@ -546,7 +587,14 @@ affine_update_priority_heaps(int32 cpu, int32 priority) int32 corePriority = AffineCorePriorityHeap::GetKey(&sCoreEntries[core]); if (corePriority != maxPriority) { - sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); + if (maxPriority == B_IDLE_PRIORITY) { + sCorePriorityHeap->ModifyKey(&sCoreEntries[core], B_IDLE_PRIORITY); + ASSERT(sCorePriorityHeap->PeekRoot() == &sCoreEntries[core]); + sCorePriorityHeap->RemoveRoot(); + } else if (corePriority == B_IDLE_PRIORITY) + sCorePriorityHeap->Insert(&sCoreEntries[core], maxPriority); + else + sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); int32 package = sCPUToPackage[cpu]; PackageEntry* packageEntry = &sPackageEntries[package]; @@ -623,8 +671,11 @@ affine_choose_core_performance(int32 priority) } else { // no idle cores, use least occupied core entry = sCorePriorityHeap->PeekRoot(); - if (AffineCorePriorityHeap::GetKey(entry) >= priority) - entry = sCoreThreadHeap->PeekRoot(); + if (AffineCorePriorityHeap::GetKey(entry) >= priority) { + entry = sCoreThreadHeap->PeekMinimum(); + if (entry == NULL) + entry = sCoreCPUBoundThreadHeap->PeekMinimum(); + } } ASSERT(entry != NULL); @@ -637,8 +688,14 @@ affine_choose_core_power_saving(int32 priority) { CoreEntry* entry; - // TODO: small tasks packing - if (sPackageUsageHeap->PeekMinimum() != NULL) { + entry = sCorePriorityHeap->PeekRoot(); + if (entry != NULL && AffineCorePriorityHeap::GetKey(entry) < priority) { + // run immediately on already woken core + } else if (sDisableSmallTaskPacking < system_time() + && sCoreThreadHeap->PeekMaximum() != NULL) { + // try to pack all threads on one core + entry = sCoreThreadHeap->PeekMaximum(); + } else if (sPackageUsageHeap->PeekMinimum() != NULL) { // wake new core PackageEntry* package = sPackageUsageHeap->PeekMinimum(); entry = package->fIdleCores.Last(); @@ -648,9 +705,9 @@ affine_choose_core_power_saving(int32 priority) entry = package->fIdleCores.Last(); } else { // no idle cores, use least occupied core - entry = sCorePriorityHeap->PeekRoot(); - if (AffineCorePriorityHeap::GetKey(entry) >= priority) - entry = sCoreThreadHeap->PeekRoot(); + entry = sCoreThreadHeap->PeekMinimum(); + if (entry == NULL) + entry = sCoreCPUBoundThreadHeap->PeekMinimum(); } ASSERT(entry != NULL); @@ -698,9 +755,11 @@ affine_should_rebalance(Thread* thread) // this one to someone less busy. int32 averageThread = sAssignedThreads / sRunQueueCount; if (coreEntry->fCPUBoundThreads > 0) { - CoreEntry* other = sCoreThreadHeap->PeekRoot(); - if (AffineCoreThreadHeap::GetKey(other) <= averageThread) + CoreEntry* other = sCoreThreadHeap->PeekMinimum(); + if (other != NULL + && AffineCoreThreadHeap::GetKey(other) <= averageThread) { return true; + } } // No cpu bound threads - the situation is quite good. Make sure it @@ -767,6 +826,8 @@ affine_thread_goes_away(Thread* thread) static void affine_enqueue(Thread* thread, bool newOne) { + ASSERT(thread != NULL); + thread->state = thread->next_state = B_THREAD_READY; scheduler_thread_data* schedulerThreadData = thread->scheduler_data; @@ -813,6 +874,9 @@ affine_enqueue(Thread* thread, bool newOne) affine_assign_active_thread_to_core(thread); } + ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); + ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); + TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, threadPriority, targetCore); if (pinned) @@ -895,17 +959,17 @@ affine_set_thread_priority(Thread *thread, int32 priority) thread->id, priority, thread->priority, affine_get_effective_priority(thread)); - if (thread->state == B_THREAD_RUNNING) { + if (thread->state == B_THREAD_RUNNING) affine_thread_goes_away(thread); - affine_update_priority_heaps(thread->cpu->cpu_num, priority); - } if (thread->state != B_THREAD_READY) { affine_cancel_penalty(thread); thread->priority = priority; - if (thread->state == B_THREAD_RUNNING) + if (thread->state == B_THREAD_RUNNING) { affine_assign_active_thread_to_core(thread); + affine_update_priority_heaps(thread->cpu->cpu_num, priority); + } return; } @@ -1403,7 +1467,7 @@ scheduler_affine_init() ArrayDeleter coreEntriesDeleter( sCoreEntries); - sCorePriorityHeap = new AffineCorePriorityHeap; + sCorePriorityHeap = new AffineCorePriorityHeap(coreCount); if (sCorePriorityHeap == NULL) return B_NO_MEMORY; ObjectDeleter corePriorityHeapDeleter( @@ -1414,20 +1478,26 @@ scheduler_affine_init() return B_NO_MEMORY; ObjectDeleter coreThreadHeapDeleter(sCoreThreadHeap); + sCoreCPUBoundThreadHeap = new AffineCoreThreadHeap(coreCount); + if (sCoreCPUBoundThreadHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter coreCPUThreadHeapDeleter( + sCoreCPUBoundThreadHeap); + for (int32 i = 0; i < coreCount; i++) { sCoreEntries[i].fCoreID = i; sCoreEntries[i].fActiveTime = 0; sCoreEntries[i].fThreads = 0; sCoreEntries[i].fCPUBoundThreads = 0; - status_t result = sCorePriorityHeap->Insert(&sCoreEntries[i], - B_IDLE_PRIORITY); + status_t result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); if (result != B_OK) return result; - result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); + result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; + sCorePriorityHeap->RemoveRoot(); } sCPUPriorityHeaps = new AffineCPUHeap[coreCount]; @@ -1477,7 +1547,11 @@ scheduler_affine_init() return result; } +#if 1 + affine_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); +#else affine_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); +#endif gScheduler = &kAffineOps; add_debugger_command_etc("run_queue", &dump_run_queue, @@ -1490,6 +1564,7 @@ scheduler_affine_init() runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); + coreCPUThreadHeapDeleter.Detach(); coreThreadHeapDeleter.Detach(); corePriorityHeapDeleter.Detach(); cpuPriorityHeapDeleter.Detach(); From 4c4994435d83053a5b4536dabab8a755af97d59c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 22 Oct 2013 23:56:31 +0200 Subject: [PATCH 066/273] kernel/util: Fixes in [MinMax]Heap implementation --- headers/private/kernel/util/Heap.h | 4 +++- headers/private/kernel/util/MinMaxHeap.h | 14 +++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h index 5d0f896ab3..ed7294c2b2 100644 --- a/headers/private/kernel/util/Heap.h +++ b/headers/private/kernel/util/Heap.h @@ -230,6 +230,7 @@ HEAP_CLASS_NAME::RemoveRoot() #if KDEBUG Element* element = PeekRoot(); HeapLink* link = sGetLink(element); + ASSERT(link->fIndex != -1); link->fIndex = -1; #endif @@ -257,6 +258,8 @@ HEAP_CLASS_NAME::Insert(Element* element, Key key) HeapLink* link = sGetLink(element); + ASSERT(link->fIndex == -1); + fElements[fLastElement] = element; link->fIndex = fLastElement++; link->fKey = key; @@ -270,7 +273,6 @@ HEAP_TEMPLATE_LIST status_t HEAP_CLASS_NAME::_GrowHeap(int minimalSize) { - minimalSize = minimalSize % 2 ? minimalSize : minimalSize + 1; int newSize = max_c(max_c(fSize * 2, 4), minimalSize); size_t arraySize = newSize * sizeof(Element*); diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 0e9f40bb2d..4dc8b0c9f0 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -258,6 +258,7 @@ MIN_MAX_HEAP_CLASS_NAME::RemoveMinimum() #if KDEBUG Element* element = PeekMinimum(); MinMaxHeapLink* link = sGetLink(element); + ASSERT(link->fIndex != -1); link->fIndex = -1; #endif @@ -278,6 +279,7 @@ MIN_MAX_HEAP_CLASS_NAME::RemoveMaximum() #if KDEBUG Element* element = PeekMaximum(); MinMaxHeapLink* link = sGetLink(element); + ASSERT(link->fIndex != -1); link->fIndex = -1; #endif @@ -296,10 +298,12 @@ MIN_MAX_HEAP_CLASS_NAME::Insert(Element* element, Key key) return result; } - ASSERT(fMinLastElement != fSize || fMaxLastElement != fSize); + ASSERT(fMinLastElement < fSize || fMaxLastElement < fSize); MinMaxHeapLink* link = sGetLink(element); + ASSERT(link->fIndex == -1); + link->fMinTree = fMinLastElement < fMaxLastElement; int& lastElement = link->fMinTree ? fMinLastElement : fMaxLastElement; @@ -320,7 +324,7 @@ MIN_MAX_HEAP_TEMPLATE_LIST status_t MIN_MAX_HEAP_CLASS_NAME::_GrowHeap(int minimalSize) { - minimalSize = minimalSize % 2 ? minimalSize : minimalSize + 1; + minimalSize = minimalSize % 2 == 0 ? minimalSize : minimalSize + 1; int newSize = max_c(max_c(fSize * 4, 4), minimalSize); size_t arraySize = newSize * sizeof(Element*); @@ -329,7 +333,11 @@ MIN_MAX_HEAP_CLASS_NAME::_GrowHeap(int minimalSize) if (newBuffer == NULL) return B_NO_MEMORY; fMinElements = newBuffer; - fMaxElements = newBuffer + (newSize / 2); + + newBuffer += newSize / 2; + if (fMaxElements != NULL) + memcpy(newBuffer, fMaxElements, fSize * sizeof(Element*)); + fMaxElements = newBuffer; fSize = newSize / 2; return B_OK; From 2df11d8a80ac3e251b655c780a5f6d3375fafab4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 23 Oct 2013 01:58:49 +0200 Subject: [PATCH 067/273] scheduler_affine: Put small tasks on a single core --- .../kernel/scheduler/scheduler_affine.cpp | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index c4b2f67e27..8c94bf4f00 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -51,10 +51,11 @@ const bigtime_t kMaxThreadQuantum = 10000; const bigtime_t kCacheExpire = 100000; static bigtime_t sDisableSmallTaskPacking; +static int32 sSmallTaskCore = -1; static scheduler_mode sSchedulerMode; -static int32 (*sChooseCore)(int32 priority); +static int32 (*sChooseCore)(Thread* thread); // Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical @@ -511,10 +512,12 @@ affine_update_thread_heaps(int32 core) if (newKey > thread_max_threads()) { if (oldKey <= thread_max_threads()) { + if (sSmallTaskCore == entry->fCoreID) + sSmallTaskCore = -1; + sCoreThreadHeap->ModifyKey(entry, -1); ASSERT(sCoreThreadHeap->PeekMinimum() == entry); sCoreThreadHeap->RemoveMinimum(); - ASSERT(sCoreThreadHeap->PeekMinimum() != entry); sCoreCPUBoundThreadHeap->Insert(entry, newKey); } else @@ -656,7 +659,7 @@ affine_update_priority_heaps(int32 cpu, int32 priority) static int32 -affine_choose_core_performance(int32 priority) +affine_choose_core_performance(Thread* thread) { CoreEntry* entry; @@ -671,6 +674,8 @@ affine_choose_core_performance(int32 priority) } else { // no idle cores, use least occupied core entry = sCorePriorityHeap->PeekRoot(); + + int32 priority = affine_get_effective_priority(thread); if (AffineCorePriorityHeap::GetKey(entry) >= priority) { entry = sCoreThreadHeap->PeekMinimum(); if (entry == NULL) @@ -684,17 +689,24 @@ affine_choose_core_performance(int32 priority) static int32 -affine_choose_core_power_saving(int32 priority) +affine_choose_core_power_saving(Thread* thread) { CoreEntry* entry; - entry = sCorePriorityHeap->PeekRoot(); - if (entry != NULL && AffineCorePriorityHeap::GetKey(entry) < priority) { - // run immediately on already woken core - } else if (sDisableSmallTaskPacking < system_time() + int32 priority = affine_get_effective_priority(thread); + int32 penalty = thread->scheduler_data->priority_penalty; + + if (!sDisableSmallTaskPacking && penalty == 0 && sCoreThreadHeap->PeekMaximum() != NULL) { // try to pack all threads on one core - entry = sCoreThreadHeap->PeekMaximum(); + if (sSmallTaskCore < 0) + sSmallTaskCore = sCoreThreadHeap->PeekMaximum()->fCoreID; + entry = &sCoreEntries[sSmallTaskCore]; + } else if (sCorePriorityHeap->PeekRoot() != NULL + && AffineCorePriorityHeap::GetKey(sCorePriorityHeap->PeekRoot()) + < priority) { + // run immediately on already woken core + entry = sCorePriorityHeap->PeekRoot(); } else if (sPackageUsageHeap->PeekMinimum() != NULL) { // wake new core PackageEntry* package = sPackageUsageHeap->PeekMinimum(); @@ -716,9 +728,9 @@ affine_choose_core_power_saving(int32 priority) static inline int32 -affine_choose_core(int32 priority) +affine_choose_core(Thread* thread) { - return sChooseCore(priority); + return sChooseCore(thread); } @@ -746,9 +758,7 @@ affine_should_rebalance(Thread* thread) // than the average get rid of this one. if (schedulerThreadData->additional_penalty != 0) { int32 averageCPUBound = sCPUBoundThreads / sRunQueueCount; - if (coreEntry->fCPUBoundThreads - averageCPUBound > 1) - return true; - return false; + return coreEntry->fCPUBoundThreads - averageCPUBound > 1; } // If this thread is not cpu bound but we have at least one consider giving @@ -762,6 +772,11 @@ affine_should_rebalance(Thread* thread) } } + // Try our luck at small task packing. + int32 penalty = schedulerThreadData->priority_penalty; + if (!sDisableSmallTaskPacking && penalty == 0) + return sSmallTaskCore != coreEntry->fCoreID; + // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... const int32 kBalanceThreshold = 3; @@ -861,7 +876,7 @@ affine_enqueue(Thread* thread, bool newOne) if (!newOne) affine_thread_goes_away(thread); - targetCore = affine_choose_core(threadPriority); + targetCore = affine_choose_core(thread); targetCPU = affine_choose_cpu(targetCore); } @@ -1325,10 +1340,12 @@ affine_set_operation_mode(scheduler_mode mode) sSchedulerMode = mode; switch (mode) { case SCHEDULER_MODE_PERFORMANCE: + sDisableSmallTaskPacking = true; sChooseCore = affine_choose_core_performance; break; case SCHEDULER_MODE_POWER_SAVING: + sDisableSmallTaskPacking = false; sChooseCore = affine_choose_core_power_saving; break; From 453bf750274a240ade4738e2e33975ee73b3384f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 23 Oct 2013 21:21:59 +0200 Subject: [PATCH 068/273] scheduler_affine: Try not to get overloaded by small tasks --- .../kernel/scheduler/scheduler_affine.cpp | 91 +++++++++++++++---- 1 file changed, 74 insertions(+), 17 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 8c94bf4f00..37208fc639 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -50,8 +50,8 @@ const bigtime_t kMaxThreadQuantum = 10000; const bigtime_t kCacheExpire = 100000; -static bigtime_t sDisableSmallTaskPacking; -static int32 sSmallTaskCore = -1; +static int sDisableSmallTaskPacking; +static int32 sSmallTaskCore; static scheduler_mode sSchedulerMode; @@ -512,9 +512,6 @@ affine_update_thread_heaps(int32 core) if (newKey > thread_max_threads()) { if (oldKey <= thread_max_threads()) { - if (sSmallTaskCore == entry->fCoreID) - sSmallTaskCore = -1; - sCoreThreadHeap->ModifyKey(entry, -1); ASSERT(sCoreThreadHeap->PeekMinimum() == entry); sCoreThreadHeap->RemoveMinimum(); @@ -535,6 +532,18 @@ affine_update_thread_heaps(int32 core) } +static inline void +affine_disable_small_task_packing(void) +{ + ASSERT(sDisableSmallTaskPacking == 0); + ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); + + ASSERT(sAssignedThreads > 0); + sDisableSmallTaskPacking = sAssignedThreads * 64; + sSmallTaskCore = -1; +} + + static inline void affine_increase_penalty(Thread* thread) { @@ -549,16 +558,26 @@ affine_increase_penalty(Thread* thread) int32 oldPenalty = schedulerThreadData->priority_penalty++; ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); + const int kMinimalPriority = affine_get_minimal_priority(thread); if (thread->priority - oldPenalty <= kMinimalPriority) { int32 core = schedulerThreadData->previous_core; ASSERT(core >= 0); - if (schedulerThreadData->additional_penalty == 0) { + + int32 additionalPenalty = schedulerThreadData->additional_penalty; + if (additionalPenalty == 0) { sCPUBoundThreads++; sCoreEntries[core].fCPUBoundThreads++; + affine_update_thread_heaps(core); } + const int kSmallTaskThreshold = 50; + if (additionalPenalty > kSmallTaskThreshold) { + if (sSmallTaskCore == core) + affine_disable_small_task_packing(); + } + schedulerThreadData->priority_penalty = oldPenalty; schedulerThreadData->additional_penalty++; } @@ -573,8 +592,22 @@ affine_cancel_penalty(Thread* thread) if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); - schedulerThreadData->priority_penalty = 0; - schedulerThreadData->additional_penalty = 0; + switch (sSchedulerMode) { + case SCHEDULER_MODE_PERFORMANCE: + schedulerThreadData->additional_penalty = 0; + schedulerThreadData->priority_penalty = 0; + break; + + case SCHEDULER_MODE_POWER_SAVING: + if (schedulerThreadData->additional_penalty != 0) + schedulerThreadData->additional_penalty /= 2; + else if (schedulerThreadData->priority_penalty != 0) + schedulerThreadData->priority_penalty--; + break; + + default: + break; + } } @@ -688,15 +721,26 @@ affine_choose_core_performance(Thread* thread) } +static inline bool +affine_is_task_small(Thread* thread) +{ + int32 priority = affine_get_effective_priority(thread); + int32 penalty = thread->scheduler_data->priority_penalty; + return penalty == 0 || priority >= B_DISPLAY_PRIORITY; +} + + static int32 affine_choose_core_power_saving(Thread* thread) { CoreEntry* entry; int32 priority = affine_get_effective_priority(thread); - int32 penalty = thread->scheduler_data->priority_penalty; - if (!sDisableSmallTaskPacking && penalty == 0 + if (sDisableSmallTaskPacking > 0) + sDisableSmallTaskPacking--; + + if (!sDisableSmallTaskPacking && affine_is_task_small(thread) && sCoreThreadHeap->PeekMaximum() != NULL) { // try to pack all threads on one core if (sSmallTaskCore < 0) @@ -772,15 +816,27 @@ affine_should_rebalance(Thread* thread) } } + int32 threadsAboveAverage = coreEntry->fThreads - averageThread; + + // All cores try to give us small tasks, check whether we have enough. + const int kSmallTaskCountThreshold = 5; + if (sDisableSmallTaskPacking == 0 && sSmallTaskCore == coreEntry->fCoreID) { + if (threadsAboveAverage > kSmallTaskCountThreshold) { + if (!affine_is_task_small(thread)) + return true; + } else if (threadsAboveAverage > 2 * kSmallTaskCountThreshold) { + affine_disable_small_task_packing(); + } + } + // Try our luck at small task packing. - int32 penalty = schedulerThreadData->priority_penalty; - if (!sDisableSmallTaskPacking && penalty == 0) + if (sDisableSmallTaskPacking == 0 && affine_is_task_small(thread)) return sSmallTaskCore != coreEntry->fCoreID; // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... const int32 kBalanceThreshold = 3; - return coreEntry->fThreads - averageThread > kBalanceThreshold; + return threadsAboveAverage > kBalanceThreshold; } @@ -1340,12 +1396,14 @@ affine_set_operation_mode(scheduler_mode mode) sSchedulerMode = mode; switch (mode) { case SCHEDULER_MODE_PERFORMANCE: - sDisableSmallTaskPacking = true; + sDisableSmallTaskPacking = -1; + sSmallTaskCore = -1; sChooseCore = affine_choose_core_performance; break; case SCHEDULER_MODE_POWER_SAVING: - sDisableSmallTaskPacking = false; + sDisableSmallTaskPacking = 0; + sSmallTaskCore = -1; sChooseCore = affine_choose_core_power_saving; break; @@ -1481,8 +1539,7 @@ scheduler_affine_init() sCoreEntries = new CoreEntry[coreCount]; if (sCoreEntries == NULL) return B_NO_MEMORY; - ArrayDeleter coreEntriesDeleter( - sCoreEntries); + ArrayDeleter coreEntriesDeleter(sCoreEntries); sCorePriorityHeap = new AffineCorePriorityHeap(coreCount); if (sCorePriorityHeap == NULL) From d6efe8ee75c6efaf53c479c028f8e158e385ff2c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 23 Oct 2013 21:56:14 +0200 Subject: [PATCH 069/273] kernel: Update cpu_ent::active_time atomically --- src/system/kernel/cpu.cpp | 7 +------ src/system/kernel/scheduler/scheduler_affine.cpp | 2 +- src/system/kernel/scheduler/scheduler_simple.cpp | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 36df9aa7cc..bacc8a3ec7 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -74,12 +74,7 @@ cpu_get_active_time(int32 cpu) if (cpu < 0 || cpu > smp_get_num_cpus()) return 0; - // We need to grab the scheduler lock here, because the thread activity - // time is not maintained atomically (because there is no need to). - - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - - return gCPU[cpu].active_time; + return atomic_get64(&gCPU[cpu].active_time); } diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index 37208fc639..bf1652b59c 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -1213,7 +1213,7 @@ affine_track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) + (oldThread->user_time - oldThread->cpu->last_user_time); - oldThread->cpu->active_time += active; + atomic_add64(&oldThread->cpu->active_time, active); sCoreEntries[thisCore].fActiveTime += active; } diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp index e4d413b2b1..c2b2e3aedb 100644 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ b/src/system/kernel/scheduler/scheduler_simple.cpp @@ -634,9 +634,9 @@ simple_reschedule(void) // track CPU activity if (!thread_is_idle_thread(oldThread)) { - oldThread->cpu->active_time += + atomic_add64(&oldThread->cpu->active_time, (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time); + + (oldThread->user_time - oldThread->cpu->last_user_time)); } if (!thread_is_idle_thread(nextThread)) { From 31a75d402f761a1087c449e44926a6bec0c31009 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 24 Oct 2013 00:01:18 +0200 Subject: [PATCH 070/273] kernel: Protect lock internals with per-lock spinlock --- headers/private/kernel/lock.h | 42 +++++------ src/system/kernel/locks/lock.cpp | 121 ++++++++++++++++++++++--------- 2 files changed, 105 insertions(+), 58 deletions(-) diff --git a/headers/private/kernel/lock.h b/headers/private/kernel/lock.h index 2a52c441d6..bf4c039566 100644 --- a/headers/private/kernel/lock.h +++ b/headers/private/kernel/lock.h @@ -18,6 +18,7 @@ struct mutex_waiter; typedef struct mutex { const char* name; struct mutex_waiter* waiters; + spinlock lock; #if KDEBUG thread_id holder; #else @@ -44,6 +45,7 @@ struct rw_lock_waiter; typedef struct rw_lock { const char* name; struct rw_lock_waiter* waiters; + spinlock lock; thread_id holder; vint32 count; int32 owner_count; @@ -88,14 +90,17 @@ typedef struct rw_lock { // static initializers #if KDEBUG -# define MUTEX_INITIALIZER(name) { name, NULL, -1, 0 } +# define MUTEX_INITIALIZER(name) \ + { name, NULL, B_SPINLOCK_INITIALIZER, -1, 0 } # define RECURSIVE_LOCK_INITIALIZER(name) { MUTEX_INITIALIZER(name), 0 } #else -# define MUTEX_INITIALIZER(name) { name, NULL, 0, 0, 0 } +# define MUTEX_INITIALIZER(name) \ + { name, NULL, B_SPINLOCK_INITIALIZER, 0, 0, 0 } # define RECURSIVE_LOCK_INITIALIZER(name) { MUTEX_INITIALIZER(name), -1, 0 } #endif -#define RW_LOCK_INITIALIZER(name) { name, NULL, -1, 0, 0, 0 } +#define RW_LOCK_INITIALIZER(name) \ + { name, NULL, B_SPINLOCK_INITIALIZER, -1, 0, 0, 0 } #if KDEBUG @@ -144,11 +149,11 @@ extern status_t mutex_switch_from_read_lock(rw_lock* from, mutex* to); extern status_t _rw_lock_read_lock(rw_lock* lock); extern status_t _rw_lock_read_lock_with_timeout(rw_lock* lock, uint32 timeoutFlags, bigtime_t timeout); -extern void _rw_lock_read_unlock(rw_lock* lock, bool schedulerLocked); -extern void _rw_lock_write_unlock(rw_lock* lock, bool schedulerLocked); +extern void _rw_lock_read_unlock(rw_lock* lock); +extern void _rw_lock_write_unlock(rw_lock* lock); -extern status_t _mutex_lock(mutex* lock, bool schedulerLocked); -extern void _mutex_unlock(mutex* lock, bool schedulerLocked); +extern status_t _mutex_lock(mutex* lock, void* locker); +extern void _mutex_unlock(mutex* lock); extern status_t _mutex_trylock(mutex* lock); extern status_t _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout); @@ -191,7 +196,7 @@ rw_lock_read_unlock(rw_lock* lock) #else int32 oldCount = atomic_add(&lock->count, -1); if (oldCount >= RW_LOCK_WRITER_COUNT_BASE) - _rw_lock_read_unlock(lock, false); + _rw_lock_read_unlock(lock); #endif } @@ -199,7 +204,7 @@ rw_lock_read_unlock(rw_lock* lock) static inline void rw_lock_write_unlock(rw_lock* lock) { - _rw_lock_write_unlock(lock, false); + _rw_lock_write_unlock(lock); } @@ -207,23 +212,10 @@ static inline status_t mutex_lock(mutex* lock) { #if KDEBUG - return _mutex_lock(lock, false); + return _mutex_lock(lock, NULL); #else if (atomic_add(&lock->count, -1) < 0) - return _mutex_lock(lock, false); - return B_OK; -#endif -} - - -static inline status_t -mutex_lock_threads_locked(mutex* lock) -{ -#if KDEBUG - return _mutex_lock(lock, true); -#else - if (atomic_add(&lock->count, -1) < 0) - return _mutex_lock(lock, true); + return _mutex_lock(lock, NULL); return B_OK; #endif } @@ -261,7 +253,7 @@ mutex_unlock(mutex* lock) #if !KDEBUG if (atomic_add(&lock->count, 1) < -1) #endif - _mutex_unlock(lock, false); + _mutex_unlock(lock); } diff --git a/src/system/kernel/locks/lock.cpp b/src/system/kernel/locks/lock.cpp index caebd36c5b..3b1d069c44 100644 --- a/src/system/kernel/locks/lock.cpp +++ b/src/system/kernel/locks/lock.cpp @@ -149,7 +149,7 @@ recursive_lock_unlock(recursive_lock *lock) static status_t -rw_lock_wait(rw_lock* lock, bool writer) +rw_lock_wait(rw_lock* lock, bool writer, InterruptsSpinLocker& locker) { // enqueue in waiter list rw_lock_waiter waiter; @@ -166,7 +166,14 @@ rw_lock_wait(rw_lock* lock, bool writer) // block thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_RW_LOCK, lock); - return thread_block_locked(waiter.thread); + locker.Unlock(); + + InterruptsSpinLocker schedulerLocker(gSchedulerLock); + status_t result = thread_block_locked(thread_get_current_thread()); + schedulerLocker.Unlock(); + + locker.Lock(); + return result; } @@ -192,7 +199,10 @@ rw_lock_unblock(rw_lock* lock) lock->holder = waiter->thread->id; // unblock thread + InterruptsSpinLocker schedulerLocker(gSchedulerLock); thread_unblock_locked(waiter->thread, B_OK); + schedulerLocker.Unlock(); + waiter->thread = NULL; return RW_LOCK_WRITER_COUNT_BASE; } @@ -208,7 +218,10 @@ rw_lock_unblock(rw_lock* lock) readerCount++; // unblock thread + InterruptsSpinLocker schedulerLocker(gSchedulerLock); thread_unblock_locked(waiter->thread, B_OK); + schedulerLocker.Unlock(); + waiter->thread = NULL; } while ((waiter = lock->waiters) != NULL && !waiter->writer); @@ -224,6 +237,7 @@ rw_lock_init(rw_lock* lock, const char* name) { lock->name = name; lock->waiters = NULL; + lock->lock = B_SPINLOCK_INITIALIZER; lock->holder = -1; lock->count = 0; lock->owner_count = 0; @@ -241,6 +255,7 @@ rw_lock_init_etc(rw_lock* lock, const char* name, uint32 flags) { lock->name = (flags & RW_LOCK_FLAG_CLONE_NAME) != 0 ? strdup(name) : name; lock->waiters = NULL; + lock->lock = B_SPINLOCK_INITIALIZER; lock->holder = -1; lock->count = 0; lock->owner_count = 0; @@ -260,7 +275,7 @@ rw_lock_destroy(rw_lock* lock) ? (char*)lock->name : NULL; // unblock all waiters - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(lock->lock); #if KDEBUG if (lock->waiters != NULL && thread_get_current_thread_id() @@ -280,6 +295,7 @@ rw_lock_destroy(rw_lock* lock) lock->waiters = waiter->next; // unblock thread + InterruptsSpinLocker _(gSchedulerLock); thread_unblock_locked(waiter->thread, B_ERROR); } @@ -296,7 +312,7 @@ rw_lock_destroy(rw_lock* lock) status_t _rw_lock_read_lock(rw_lock* lock) { - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(lock->lock); // We might be the writer ourselves. if (lock->holder == thread_get_current_thread_id()) { @@ -320,7 +336,7 @@ _rw_lock_read_lock(rw_lock* lock) ASSERT(lock->count >= RW_LOCK_WRITER_COUNT_BASE); // we need to wait - return rw_lock_wait(lock, false); + return rw_lock_wait(lock, false, locker); } @@ -328,7 +344,7 @@ status_t _rw_lock_read_lock_with_timeout(rw_lock* lock, uint32 timeoutFlags, bigtime_t timeout) { - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(lock->lock); // We might be the writer ourselves. if (lock->holder == thread_get_current_thread_id()) { @@ -368,13 +384,19 @@ _rw_lock_read_lock_with_timeout(rw_lock* lock, uint32 timeoutFlags, // block thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_RW_LOCK, lock); + locker.Unlock(); + + InterruptsSpinLocker schedulerLock(gSchedulerLock); status_t error = thread_block_with_timeout_locked(timeoutFlags, timeout); + schedulerLock.Unlock(); + if (error == B_OK || waiter.thread == NULL) { // We were unblocked successfully -- potentially our unblocker overtook // us after we already failed. In either case, we've got the lock, now. return B_OK; } + locker.Lock(); // We failed to get the lock -- dequeue from waiter list. rw_lock_waiter* previous = NULL; rw_lock_waiter* other = lock->waiters; @@ -407,9 +429,9 @@ _rw_lock_read_lock_with_timeout(rw_lock* lock, uint32 timeoutFlags, void -_rw_lock_read_unlock(rw_lock* lock, bool schedulerLocked) +_rw_lock_read_unlock(rw_lock* lock) { - InterruptsSpinLocker locker(gSchedulerLock, false, !schedulerLocked); + InterruptsSpinLocker locker(lock->lock); // If we're still holding the write lock or if there are other readers, // no-one can be woken up. @@ -437,7 +459,7 @@ _rw_lock_read_unlock(rw_lock* lock, bool schedulerLocked) status_t rw_lock_write_lock(rw_lock* lock) { - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(lock->lock); // If we're already the lock holder, we just need to increment the owner // count. @@ -462,7 +484,7 @@ rw_lock_write_lock(rw_lock* lock) if (oldCount < RW_LOCK_WRITER_COUNT_BASE) lock->active_readers = oldCount - lock->pending_readers; - status_t status = rw_lock_wait(lock, true); + status_t status = rw_lock_wait(lock, true, locker); if (status == B_OK) { lock->holder = thread; lock->owner_count = RW_LOCK_WRITER_COUNT_BASE; @@ -473,9 +495,9 @@ rw_lock_write_lock(rw_lock* lock) void -_rw_lock_write_unlock(rw_lock* lock, bool schedulerLocked) +_rw_lock_write_unlock(rw_lock* lock) { - InterruptsSpinLocker locker(gSchedulerLock, false, !schedulerLocked); + InterruptsSpinLocker locker(lock->lock); if (thread_get_current_thread_id() != lock->holder) { panic("rw_lock_write_unlock(): lock %p not write-locked by this thread", @@ -562,6 +584,7 @@ mutex_init(mutex* lock, const char *name) { lock->name = name; lock->waiters = NULL; + lock->lock = B_SPINLOCK_INITIALIZER; #if KDEBUG lock->holder = -1; #else @@ -580,6 +603,7 @@ mutex_init_etc(mutex* lock, const char *name, uint32 flags) { lock->name = (flags & MUTEX_FLAG_CLONE_NAME) != 0 ? strdup(name) : name; lock->waiters = NULL; + lock->lock = B_SPINLOCK_INITIALIZER; #if KDEBUG lock->holder = -1; #else @@ -600,15 +624,16 @@ mutex_destroy(mutex* lock) ? (char*)lock->name : NULL; // unblock all waiters - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(lock->lock); #if KDEBUG if (lock->waiters != NULL && thread_get_current_thread_id() != lock->holder) { panic("mutex_destroy(): there are blocking threads, but caller doesn't " "hold the lock (%p)", lock); - if (_mutex_lock(lock, true) != B_OK) + if (_mutex_lock(lock, &locker) != B_OK) return; + locker.Lock(); } #endif @@ -617,6 +642,7 @@ mutex_destroy(mutex* lock) lock->waiters = waiter->next; // unblock thread + InterruptsSpinLocker schedulerLocker(gSchedulerLock); thread_unblock_locked(waiter->thread, B_ERROR); } @@ -628,49 +654,69 @@ mutex_destroy(mutex* lock) } +static inline status_t +mutex_lock_threads_locked(mutex* lock, InterruptsSpinLocker* locker) +{ +#if KDEBUG + return _mutex_lock(lock, locker); +#else + if (atomic_add(&lock->count, -1) < 0) + return _mutex_lock(lock, locker); + return B_OK; +#endif +} + + status_t mutex_switch_lock(mutex* from, mutex* to) { - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(to->lock); #if !KDEBUG if (atomic_add(&from->count, 1) < -1) #endif - _mutex_unlock(from, true); + _mutex_unlock(from); - return mutex_lock_threads_locked(to); + return mutex_lock_threads_locked(to, &locker); } status_t mutex_switch_from_read_lock(rw_lock* from, mutex* to) { - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(to->lock); #if KDEBUG_RW_LOCK_DEBUG - _rw_lock_write_unlock(from, true); + _rw_lock_write_unlock(from); #else int32 oldCount = atomic_add(&from->count, -1); if (oldCount >= RW_LOCK_WRITER_COUNT_BASE) - _rw_lock_read_unlock(from, true); + _rw_lock_read_unlock(from); #endif - return mutex_lock_threads_locked(to); + return mutex_lock_threads_locked(to, &locker); } status_t -_mutex_lock(mutex* lock, bool schedulerLocked) +_mutex_lock(mutex* lock, void* _locker) { #if KDEBUG - if (!gKernelStartup && !schedulerLocked && !are_interrupts_enabled()) { + if (!gKernelStartup && _locker == NULL && !are_interrupts_enabled()) { panic("_mutex_lock(): called with interrupts disabled for lock %p", lock); } #endif - // lock only, if !threadsLocked - InterruptsSpinLocker locker(gSchedulerLock, false, !schedulerLocked); + // lock only, if !lockLocked + InterruptsSpinLocker* locker + = reinterpret_cast(_locker); + + InterruptsSpinLocker lockLocker; + if (locker == NULL) { + lockLocker.SetTo(lock->lock, false); + locker = &lockLocker; + } // Might have been released after we decremented the count, but before // we acquired the spinlock. @@ -704,22 +750,24 @@ _mutex_lock(mutex* lock, bool schedulerLocked) // block thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_MUTEX, lock); + locker->Unlock(); + + InterruptsSpinLocker schedulerLocker(gSchedulerLock); status_t error = thread_block_locked(waiter.thread); + schedulerLocker.Unlock(); #if KDEBUG if (error == B_OK) - lock->holder = waiter.thread->id; + atomic_set(&lock->holder, waiter.thread->id); #endif - return error; } void -_mutex_unlock(mutex* lock, bool schedulerLocked) +_mutex_unlock(mutex* lock) { - // lock only, if !threadsLocked - InterruptsSpinLocker locker(gSchedulerLock, false, !schedulerLocked); + InterruptsSpinLocker locker(lock->lock); #if KDEBUG if (thread_get_current_thread_id() != lock->holder) { @@ -743,7 +791,9 @@ _mutex_unlock(mutex* lock, bool schedulerLocked) lock->waiters->last = waiter->last; // unblock thread + InterruptsSpinLocker schedulerLocker(gSchedulerLock); thread_unblock_locked(waiter->thread, B_OK); + schedulerLocker.Unlock(); #if KDEBUG // Already set the holder to the unblocked thread. Besides that this @@ -768,7 +818,7 @@ status_t _mutex_trylock(mutex* lock) { #if KDEBUG - InterruptsSpinLocker _(gSchedulerLock); + InterruptsSpinLocker _(lock->lock); if (lock->holder <= 0) { lock->holder = thread_get_current_thread_id(); @@ -789,7 +839,7 @@ _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout) } #endif - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(lock->lock); // Might have been released after we decremented the count, but before // we acquired the spinlock. @@ -823,8 +873,13 @@ _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout) // block thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_MUTEX, lock); - status_t error = thread_block_with_timeout_locked(timeoutFlags, timeout); + locker.Unlock(); + InterruptsSpinLocker schedulerLocker(gSchedulerLock); + status_t error = thread_block_with_timeout_locked(timeoutFlags, timeout); + schedulerLocker.Unlock(); + + locker.Lock(); if (error == B_OK) { #if KDEBUG lock->holder = waiter.thread->id; From 7e1ecb9315396949d3197916c1a3bf67a0fee22c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 24 Oct 2013 00:59:10 +0200 Subject: [PATCH 071/273] kernel: Protect scheduler_set_thread_priority() with lock --- src/system/kernel/thread.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 391a487fbe..d13fec1a5b 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1919,7 +1919,10 @@ thread_exit(void) panic("thread_exit() called with interrupts disabled!\n"); // boost our priority to get this over with - scheduler_set_thread_priority(thread, B_URGENT_DISPLAY_PRIORITY); + { + InterruptsSpinLocker _(gSchedulerLock); + scheduler_set_thread_priority(thread, B_URGENT_DISPLAY_PRIORITY); + } if (team != kernelTeam) { // Cancel previously installed alarm timer, if any. Hold the scheduler From ed8627e5358e6bd5b901545c79a4f58c51c838b3 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 24 Oct 2013 00:59:58 +0200 Subject: [PATCH 072/273] kernel/util: Fix MinMaxHeap::_GrowHeap() --- headers/private/kernel/util/MinMaxHeap.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 4dc8b0c9f0..14efee5ac8 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -335,8 +335,8 @@ MIN_MAX_HEAP_CLASS_NAME::_GrowHeap(int minimalSize) fMinElements = newBuffer; newBuffer += newSize / 2; - if (fMaxElements != NULL) - memcpy(newBuffer, fMaxElements, fSize * sizeof(Element*)); + if (fMaxLastElement > 0) + memcpy(newBuffer, fMinElements + fSize, fSize * sizeof(Element*)); fMaxElements = newBuffer; fSize = newSize / 2; @@ -426,7 +426,7 @@ MIN_MAX_HEAP_CLASS_NAME::_ChangeTree(MinMaxHeapLink* link) if (otherLastElement <= 0) { ASSERT(currentLastElement == 1); - return true; + return false; } ASSERT((link->fIndex - 1) / 2 < otherLastElement); From e927edd3765865bd8e7729ca15bd0ee0602be42c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 24 Oct 2013 01:33:12 +0200 Subject: [PATCH 073/273] scheduler_affine: Disable logic not needed on current topology --- .../kernel/scheduler/scheduler_affine.cpp | 41 +++++++++++++++---- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp index bf1652b59c..a0fb24e109 100644 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ b/src/system/kernel/scheduler/scheduler_affine.cpp @@ -53,6 +53,8 @@ const bigtime_t kCacheExpire = 100000; static int sDisableSmallTaskPacking; static int32 sSmallTaskCore; +static bool sSingleCore; + static scheduler_mode sSchedulerMode; static int32 (*sChooseCore)(Thread* thread); @@ -439,6 +441,8 @@ dump_idle_cores(int argc, char** argv) static inline bool affine_has_cache_expired(Thread* thread) { + ASSERT(!sSingleCore); + if (thread_is_idle_thread(thread)) return false; @@ -495,6 +499,8 @@ affine_dump_thread_data(Thread* thread) static void affine_update_thread_heaps(int32 core) { + ASSERT(!sSingleCore); + CoreEntry* entry = &sCoreEntries[core]; ASSERT(entry->fCPUBoundThreads >= 0 @@ -535,6 +541,8 @@ affine_update_thread_heaps(int32 core) static inline void affine_disable_small_task_packing(void) { + ASSERT(!sSingleCore); + ASSERT(sDisableSmallTaskPacking == 0); ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); @@ -565,7 +573,7 @@ affine_increase_penalty(Thread* thread) ASSERT(core >= 0); int32 additionalPenalty = schedulerThreadData->additional_penalty; - if (additionalPenalty == 0) { + if (additionalPenalty == 0 && !sSingleCore) { sCPUBoundThreads++; sCoreEntries[core].fCPUBoundThreads++; @@ -573,7 +581,7 @@ affine_increase_penalty(Thread* thread) } const int kSmallTaskThreshold = 50; - if (additionalPenalty > kSmallTaskThreshold) { + if (additionalPenalty > kSmallTaskThreshold && !sSingleCore) { if (sSmallTaskCore == core) affine_disable_small_task_packing(); } @@ -618,6 +626,9 @@ affine_update_priority_heaps(int32 cpu, int32 priority) sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); + if (sSingleCore) + return; + int32 maxPriority = AffineCPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); int32 corePriority = AffineCorePriorityHeap::GetKey(&sCoreEntries[core]); @@ -774,6 +785,7 @@ affine_choose_core_power_saving(Thread* thread) static inline int32 affine_choose_core(Thread* thread) { + ASSERT(!sSingleCore); return sChooseCore(thread); } @@ -790,6 +802,8 @@ affine_choose_cpu(int32 core) static bool affine_should_rebalance(Thread* thread) { + ASSERT(!sSingleCore); + if (thread_is_idle_thread(thread)) return false; @@ -843,7 +857,7 @@ affine_should_rebalance(Thread* thread) static void affine_assign_active_thread_to_core(Thread* thread) { - if (thread_is_idle_thread(thread)) + if (thread_is_idle_thread(thread) || sSingleCore) return; scheduler_thread_data* schedulerThreadData = thread->scheduler_data; @@ -874,6 +888,12 @@ affine_thread_goes_away(Thread* thread) ASSERT(schedulerThreadData->previous_core >= 0); int32 core = schedulerThreadData->previous_core; + schedulerThreadData->went_sleep = system_time(); + schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; + + if (sSingleCore) + return; + ASSERT(sCoreEntries[core].fThreads > 0); ASSERT(sCoreEntries[core].fThreads > sCoreEntries[core].fCPUBoundThreads || (sCoreEntries[core].fThreads == sCoreEntries[core].fCPUBoundThreads @@ -888,9 +908,6 @@ affine_thread_goes_away(Thread* thread) } affine_update_thread_heaps(core); - - schedulerThreadData->went_sleep = system_time(); - schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; } @@ -921,6 +938,11 @@ affine_enqueue(Thread* thread, bool newOne) if (newOne) affine_assign_active_thread_to_core(thread); + } else if (sSingleCore) { + targetCore = 0; + targetCPU = affine_choose_cpu(targetCore); + + schedulerThreadData->previous_core = targetCore; } else if (schedulerThreadData->previous_core < 0 || (newOne && affine_has_cache_expired(thread)) || affine_should_rebalance(thread)) { @@ -1506,6 +1528,7 @@ scheduler_affine_init() if (result != B_OK) return result; sRunQueueCount = coreCount; + sSingleCore = coreCount == 1; // create package heap and idle package stack sPackageEntries = new(std::nothrow) PackageEntry[packageCount]; @@ -1633,8 +1656,10 @@ scheduler_affine_init() add_debugger_command_etc("cpu_heap", &dump_cpu_heap, "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", 0); - add_debugger_command_etc("idle_cores", &dump_idle_cores, - "List idle cores", "\nList idle cores", 0); + if (!sSingleCore) { + add_debugger_command_etc("idle_cores", &dump_idle_cores, + "List idle cores", "\nList idle cores", 0); + } runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); From 978fc080654a2367cfb75a8afa196361ab56645e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 24 Oct 2013 02:04:03 +0200 Subject: [PATCH 074/273] scheduler: Remove support for running different schedulers Simple scheduler behaves exactly the same as affine scheduler with a single core. Obviously, affine scheduler is more complicated thus introduces greater overhead but quite a lot of multicore logic has been disabled on single core systems in the previous commit. --- headers/private/kernel/kscheduler.h | 130 +- src/system/kernel/Jamfile | 2 - src/system/kernel/scheduler/scheduler.cpp | 1748 ++++++++++++++++- .../kernel/scheduler/scheduler_affine.cpp | 1676 ---------------- .../kernel/scheduler/scheduler_affine.h | 13 - .../kernel/scheduler/scheduler_simple.cpp | 776 -------- .../kernel/scheduler/scheduler_simple.h | 12 - src/system/kernel/thread.cpp | 6 +- 8 files changed, 1736 insertions(+), 2627 deletions(-) delete mode 100644 src/system/kernel/scheduler/scheduler_affine.cpp delete mode 100644 src/system/kernel/scheduler/scheduler_affine.h delete mode 100644 src/system/kernel/scheduler/scheduler_simple.cpp delete mode 100644 src/system/kernel/scheduler/scheduler_simple.h diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index 7746222f38..bfdac1d7e2 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -24,86 +24,70 @@ typedef enum scheduler_mode { SCHEDULER_MODE_COUNT } scheduler_mode; -struct scheduler_ops { - /*! Enqueues the thread in the ready-to-run queue. - The caller must hold the scheduler lock (with disabled interrupts). - */ - void (*enqueue_in_run_queue)(Thread* thread); - - /*! Selects a thread from the ready-to-run queue and, if that's not the - calling thread, switches the current CPU's context to run the selected - thread. - If it's the same thread, the thread will just continue to run. - In either case, unless the thread is dead or is sleeping/waiting - indefinitely, the function will eventually return. - The caller must hold the scheduler lock (with disabled interrupts). - */ - void (*reschedule)(void); - - /*! Sets the given thread's priority. - The thread may be running or may be in the ready-to-run queue. - The caller must hold the scheduler lock (with disabled interrupts). - */ - void (*set_thread_priority)(Thread* thread, int32 priority); - bigtime_t (*estimate_max_scheduling_latency)(Thread* thread); - - /*! Called when the Thread structure is first created. - Per-thread housekeeping resources can be allocated. - Interrupts must be enabled. - */ - status_t (*on_thread_create)(Thread* thread, bool idleThread); - - /*! Called when a Thread structure is initialized and made ready for - use. - The per-thread housekeeping data structures are reset, if needed. - The caller must hold the scheduler lock (with disabled interrupts). - */ - void (*on_thread_init)(Thread* thread); - - /*! Called when a Thread structure is freed. - Frees up any per-thread resources allocated on the scheduler's part. The - function may be called even if on_thread_create() failed. - Interrupts must be enabled. - */ - void (*on_thread_destroy)(Thread* thread); - - /*! Called in the early boot process to start thread scheduling on the - current CPU. - The function is called once for each CPU. - Interrupts must be disabled, but the caller must not hold the scheduler - lock. - */ - void (*start)(void); - - /*! Sets scheduler operation mode. - */ - status_t (*set_operation_mode)(scheduler_mode mode); - - /*! Dumps scheduler specific thread information. - */ - void (*dump_thread_data)(Thread* thread); -}; - -extern struct scheduler_ops* gScheduler; extern spinlock gSchedulerLock; -#define scheduler_enqueue_in_run_queue(thread) \ - gScheduler->enqueue_in_run_queue(thread) -#define scheduler_set_thread_priority(thread, priority) \ - gScheduler->set_thread_priority(thread, priority) -#define scheduler_reschedule() gScheduler->reschedule() -#define scheduler_start() gScheduler->start() -#define scheduler_on_thread_create(thread, idleThread) \ - gScheduler->on_thread_create(thread, idleThread) -#define scheduler_on_thread_init(thread) \ - gScheduler->on_thread_init(thread) -#define scheduler_on_thread_destroy(thread) \ - gScheduler->on_thread_destroy(thread) #ifdef __cplusplus extern "C" { #endif +/*! Enqueues the thread in the ready-to-run queue. + The caller must hold the scheduler lock (with disabled interrupts). +*/ +void scheduler_enqueue_in_run_queue(Thread* thread); + +/*! Selects a thread from the ready-to-run queue and, if that's not the + calling thread, switches the current CPU's context to run the selected + thread. + If it's the same thread, the thread will just continue to run. + In either case, unless the thread is dead or is sleeping/waiting + indefinitely, the function will eventually return. + The caller must hold the scheduler lock (with disabled interrupts). +*/ +void scheduler_reschedule(void); + +/*! Sets the given thread's priority. + The thread may be running or may be in the ready-to-run queue. + The caller must hold the scheduler lock (with disabled interrupts). +*/ +void scheduler_set_thread_priority(Thread* thread, int32 priority); + +/*! Called when the Thread structure is first created. + Per-thread housekeeping resources can be allocated. + Interrupts must be enabled. +*/ +status_t scheduler_on_thread_create(Thread* thread, bool idleThread); + +/*! Called when a Thread structure is initialized and made ready for + use. + The per-thread housekeeping data structures are reset, if needed. + The caller must hold the scheduler lock (with disabled interrupts). +*/ +void scheduler_on_thread_init(Thread* thread); + +/*! Called when a Thread structure is freed. + Frees up any per-thread resources allocated on the scheduler's part. The + function may be called even if on_thread_create() failed. + Interrupts must be enabled. +*/ +void scheduler_on_thread_destroy(Thread* thread); + +/*! Called in the early boot process to start thread scheduling on the + current CPU. + The function is called once for each CPU. + Interrupts must be disabled, but the caller must not hold the scheduler + lock. +*/ +void scheduler_start(void); + +/*! Sets scheduler operation mode. + */ +status_t scheduler_set_operation_mode(scheduler_mode mode); + +/*! Dumps scheduler specific thread information. +*/ +void scheduler_dump_thread_data(Thread* thread); + void scheduler_add_listener(struct SchedulerListener* listener); void scheduler_remove_listener(struct SchedulerListener* listener); diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 2c0079bdc7..9bf77eb834 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -63,8 +63,6 @@ KernelMergeObject kernel_core.o : # scheduler scheduler.cpp - scheduler_affine.cpp - scheduler_simple.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index fd843119c3..18357d26fe 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1,32 +1,1688 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Copyright 2009, Rene Gollent, rene@gollent.com. * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. - * Copyright 2010, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2002, Angelo Mottola, a.mottola@libero.it. * Distributed under the terms of the MIT License. + * + * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. + * Distributed under the terms of the NewOS License. */ +/*! The thread scheduler */ + + +#include + +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include +#include +#include -#include "scheduler_affine.h" -#include "scheduler_simple.h" +#include "RunQueue.h" +#include "scheduler_common.h" #include "scheduler_tracing.h" -struct scheduler_ops* gScheduler; +//#define TRACE_SCHEDULER +#ifdef TRACE_SCHEDULER +# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) +#else +# define TRACE(...) do { } while (false) +#endif + + spinlock gSchedulerLock = B_SPINLOCK_INITIALIZER; SchedulerListenerList gSchedulerListeners; -static void (*sRescheduleFunction)(void); +bool sSchedulerEnabled; + +const bigtime_t kThreadQuantum = 1000; +const bigtime_t kMinThreadQuantum = 3000; +const bigtime_t kMaxThreadQuantum = 10000; + +const bigtime_t kCacheExpire = 100000; + +static int sDisableSmallTaskPacking; +static int32 sSmallTaskCore; + +static bool sSingleCore; + +static scheduler_mode sSchedulerMode; + +static int32 (*sChooseCore)(Thread* thread); + + +// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical +// processors in the heap belong to. Since there are no cache affinity issues +// at this level and the run queue is shared among all logical processors on +// the core the only real concern is to make lower priority threads give way to +// the higher priority threads. +struct CPUEntry : public MinMaxHeapLinkImpl { + int32 fCPUNumber; +}; +typedef MinMaxHeap CPUHeap; +static CPUEntry* sCPUEntries; +static CPUHeap* sCPUPriorityHeaps; + +struct CoreEntry : public DoublyLinkedListLinkImpl { + HeapLink fPriorityHeapLink; + MinMaxHeapLink fThreadHeapLink; + + int32 fCoreID; + + bigtime_t fActiveTime; + + int32 fCPUBoundThreads; + int32 fThreads; +}; + +static CoreEntry* sCoreEntries; +typedef Heap, + HeapMemberGetLink > + CorePriorityHeap; +static CorePriorityHeap* sCorePriorityHeap; + +typedef MinMaxHeap, + MinMaxHeapMemberGetLink > + CoreThreadHeap; +static CoreThreadHeap* sCoreThreadHeap; +static CoreThreadHeap* sCoreCPUBoundThreadHeap; + +static int32 sCPUBoundThreads; +static int32 sAssignedThreads; + +// sPackageUsageHeap is used to decide which core should be woken up from the +// idle state. When aiming for performance we should use as many packages as +// possible with as little cores active in each package as possible (so that the +// package can enter any boost mode if it has one and the active core have more +// of the shared cache for themselves. If power saving is the main priority we +// should keep active cores on as little packages as possible (so that other +// packages can go to the deep state of sleep). The heap stores only packages +// with at least one core active and one core idle. The packages with all cores +// idle are stored in sPackageIdleList (in LIFO manner). +struct PackageEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { + int32 fPackageID; + + DoublyLinkedList fIdleCores; + int32 fIdleCoreCount; + + int32 fCoreCount; +}; +typedef MinMaxHeap PackageHeap; +typedef DoublyLinkedList IdlePackageList; + +static PackageEntry* sPackageEntries; +static PackageHeap* sPackageUsageHeap; +static IdlePackageList* sIdlePackageList; + +// The run queues. Holds the threads ready to run ordered by priority. +// One queue per schedulable target per core. Additionally, each +// logical processor has its sPinnedRunQueues used for scheduling +// pinned threads. +typedef RunQueue ThreadRunQueue; +static ThreadRunQueue* sRunQueues; +static ThreadRunQueue* sPinnedRunQueues; +static int32 sRunQueueCount; + +// Since CPU IDs used internally by the kernel bear no relation to the actual +// CPU topology the following arrays are used to efficiently get the core +// and the package that CPU in question belongs to. +static int32* sCPUToCore; +static int32* sCPUToPackage; + + +struct scheduler_thread_data { + scheduler_thread_data() { Init(); } + inline void Init(); + + int32 priority_penalty; + int32 additional_penalty; + + bool lost_cpu; + bool cpu_bound; + + bigtime_t time_left; + bigtime_t stolen_time; + bigtime_t quantum_start; + + bigtime_t went_sleep; + bigtime_t went_sleep_active; + + int32 previous_core; +}; + + +void +scheduler_thread_data::Init() +{ + priority_penalty = 0; + additional_penalty = 0; + + time_left = 0; + stolen_time = 0; + + went_sleep = 0; + went_sleep_active = 0; + + lost_cpu = false; + cpu_bound = true; + + previous_core = -1; +} + + +static inline int +get_minimal_priority(Thread* thread) +{ + return min_c(thread->priority, 25) / 5; +} + + +static inline int32 +get_thread_penalty(Thread* thread) +{ + int32 penalty = thread->scheduler_data->priority_penalty; + + const int kMinimalPriority = get_minimal_priority(thread); + if (kMinimalPriority > 0) { + penalty + += thread->scheduler_data->additional_penalty % kMinimalPriority; + } + + return penalty; +} + + +static inline int32 +get_effective_priority(Thread* thread) +{ + if (thread->priority == B_IDLE_PRIORITY) + return thread->priority; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return thread->priority; + + int32 effectivePriority = thread->priority; + effectivePriority -= get_thread_penalty(thread); + + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return effectivePriority; +} static void -scheduler_reschedule_no_op(void) +dump_queue(ThreadRunQueue::ConstIterator& iterator) { - Thread* thread = thread_get_current_thread(); - if (thread != NULL && thread->next_state != B_THREAD_READY) - panic("scheduler_reschedule_no_op() called in non-ready thread"); + if (!iterator.HasNext()) + kprintf("Run queue is empty.\n"); + else { + kprintf("thread id priority penalty name\n"); + while (iterator.HasNext()) { + Thread* thread = iterator.Next(); + kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", + thread, thread->id, thread->priority, + get_thread_penalty(thread), thread->name); + } + } +} + + +static int +dump_run_queue(int argc, char **argv) +{ + int32 cpuCount = smp_get_num_cpus(); + int32 coreCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) + sCPUToCore[i] = coreCount++; + } + + ThreadRunQueue::ConstIterator iterator; + for (int32 i = 0; i < coreCount; i++) { + kprintf("\nCore %" B_PRId32 " run queue:\n", i); + iterator = sRunQueues[i].GetConstIterator(); + dump_queue(iterator); + } + + for (int32 i = 0; i < cpuCount; i++) { + iterator = sPinnedRunQueues[i].GetConstIterator(); + + if (iterator.HasNext()) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", i); + dump_queue(iterator); + } + } + + return 0; +} + + +static void +dump_heap(CPUHeap* heap) +{ + CPUHeap temp(smp_get_num_cpus()); + + kprintf("cpu priority actual priority\n"); + CPUEntry* entry = heap->PeekMinimum(); + while (entry) { + int32 cpu = entry->fCPUNumber; + int32 key = CPUHeap::GetKey(entry); + kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, key, + get_effective_priority(gCPU[cpu].running_thread)); + + heap->RemoveMinimum(); + temp.Insert(entry, key); + + entry = heap->PeekMinimum(); + } + + entry = temp.PeekMinimum(); + while (entry) { + int32 key = CPUHeap::GetKey(entry); + temp.RemoveMinimum(); + heap->Insert(entry, key); + entry = temp.PeekMinimum(); + } +} + + +static void +dump_core_thread_heap(CoreThreadHeap* heap) +{ + CoreThreadHeap temp(sRunQueueCount); + + CoreEntry* entry = heap->PeekMinimum(); + while (entry) { + int32 key = CoreThreadHeap::GetKey(entry); + kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 "\n", + entry->fCoreID, key, entry->fThreads, entry->fCPUBoundThreads); + + heap->RemoveMinimum(); + temp.Insert(entry, key); + + entry = heap->PeekMinimum(); + } + + entry = temp.PeekMinimum(); + while (entry) { + int32 key = CoreThreadHeap::GetKey(entry); + temp.RemoveMinimum(); + heap->Insert(entry, key); + entry = temp.PeekMinimum(); + } +} + + +static int +dump_cpu_heap(int argc, char** argv) +{ + CorePriorityHeap temp(sRunQueueCount); + + CoreEntry* entry = sCorePriorityHeap->PeekRoot(); + if (entry != NULL) + kprintf("core priority\n"); + else + kprintf("No active cores.\n"); + + while (entry) { + int32 core = entry->fCoreID; + int32 key = CorePriorityHeap::GetKey(entry); + kprintf("%4" B_PRId32 " %8" B_PRId32 "\n", core, key); + + sCorePriorityHeap->RemoveRoot(); + temp.Insert(entry, key); + + entry = sCorePriorityHeap->PeekRoot(); + } + + entry = temp.PeekRoot(); + while (entry) { + int32 key = CorePriorityHeap::GetKey(entry); + temp.RemoveRoot(); + sCorePriorityHeap->Insert(entry, key); + entry = temp.PeekRoot(); + } + + kprintf("\ncore key threads cpu-bound\n"); + dump_core_thread_heap(sCoreThreadHeap); + dump_core_thread_heap(sCoreCPUBoundThreadHeap); + + for (int32 i = 0; i < sRunQueueCount; i++) { + kprintf("\nCore %" B_PRId32 " heap:\n", i); + dump_heap(&sCPUPriorityHeaps[i]); + } + + return 0; +} + + +static int +dump_idle_cores(int argc, char** argv) +{ + kprintf("Idle packages:\n"); + IdlePackageList::ReverseIterator idleIterator + = sIdlePackageList->GetReverseIterator(); + + if (idleIterator.HasNext()) { + kprintf("package cores\n"); + + while (idleIterator.HasNext()) { + PackageEntry* entry = idleIterator.Next(); + kprintf("%-7" B_PRId32 " ", entry->fPackageID); + + DoublyLinkedList::ReverseIterator iterator + = entry->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); + } + } else + kprintf("No idle packages.\n"); + + PackageHeap temp(smp_get_num_cpus()); + kprintf("\nPackages with idle cores:\n"); + + PackageEntry* entry = sPackageUsageHeap->PeekMinimum(); + if (entry == NULL) + kprintf("No packages.\n"); + else + kprintf("package count cores\n"); + + while (entry != NULL) { + kprintf("%-7" B_PRId32 " %-5" B_PRId32 " ", entry->fPackageID, + entry->fIdleCoreCount); + + DoublyLinkedList::ReverseIterator iterator + = entry->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); + + sPackageUsageHeap->RemoveMinimum(); + temp.Insert(entry, entry->fIdleCoreCount); + + entry = sPackageUsageHeap->PeekMinimum(); + } + + entry = temp.PeekMinimum(); + while (entry != NULL) { + int32 key = PackageHeap::GetKey(entry); + temp.RemoveMinimum(); + sPackageUsageHeap->Insert(entry, key); + entry = temp.PeekMinimum(); + } + + return 0; +} + + +static inline bool +has_cache_expired(Thread* thread) +{ + ASSERT(!sSingleCore); + + if (thread_is_idle_thread(thread)) + return false; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; + switch (sSchedulerMode) { + case SCHEDULER_MODE_PERFORMANCE: + return coreEntry->fActiveTime + - schedulerThreadData->went_sleep_active > kCacheExpire; + + case SCHEDULER_MODE_POWER_SAVING: + return system_time() - schedulerThreadData->went_sleep + > kCacheExpire; + + default: + return true; + } +} + + +void +scheduler_dump_thread_data(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", + schedulerThreadData->priority_penalty); + + int32 additionalPenalty = 0; + const int kMinimalPriority = get_minimal_priority(thread); + if (kMinimalPriority > 0) { + additionalPenalty + = schedulerThreadData->additional_penalty % kMinimalPriority; + } + kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", + additionalPenalty, schedulerThreadData->additional_penalty); + kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", + schedulerThreadData->stolen_time); + kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", + schedulerThreadData->went_sleep); + kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", + schedulerThreadData->went_sleep_active); + kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", + schedulerThreadData->previous_core); + if (schedulerThreadData->previous_core > 0 + && has_cache_expired(thread)) { + kprintf("\tcache affinity has expired\n"); + } +} + + +static void +update_thread_heaps(int32 core) +{ + ASSERT(!sSingleCore); + + CoreEntry* entry = &sCoreEntries[core]; + + ASSERT(entry->fCPUBoundThreads >= 0 + && entry->fCPUBoundThreads <= entry->fThreads); + ASSERT(entry->fThreads >= 0 + && entry->fThreads <= thread_max_threads()); + + int32 newKey = entry->fCPUBoundThreads * thread_max_threads(); + newKey += entry->fThreads; + + int32 oldKey = CoreThreadHeap::GetKey(entry); + + if (oldKey == newKey) + return; + + if (newKey > thread_max_threads()) { + if (oldKey <= thread_max_threads()) { + sCoreThreadHeap->ModifyKey(entry, -1); + ASSERT(sCoreThreadHeap->PeekMinimum() == entry); + sCoreThreadHeap->RemoveMinimum(); + + sCoreCPUBoundThreadHeap->Insert(entry, newKey); + } else + sCoreCPUBoundThreadHeap->ModifyKey(entry, newKey); + } else { + if (oldKey > thread_max_threads()) { + sCoreCPUBoundThreadHeap->ModifyKey(entry, -1); + ASSERT(sCoreCPUBoundThreadHeap->PeekMinimum() == entry); + sCoreCPUBoundThreadHeap->RemoveMinimum(); + + sCoreThreadHeap->Insert(entry, newKey); + } else + sCoreThreadHeap->ModifyKey(entry, newKey); + } +} + + +static inline void +disable_small_task_packing(void) +{ + ASSERT(!sSingleCore); + + ASSERT(sDisableSmallTaskPacking == 0); + ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); + + ASSERT(sAssignedThreads > 0); + sDisableSmallTaskPacking = sAssignedThreads * 64; + sSmallTaskCore = -1; +} + + +static inline void +increase_penalty(Thread* thread) +{ + if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) + return; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return; + + TRACE("increasing thread %ld penalty\n", thread->id); + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + int32 oldPenalty = schedulerThreadData->priority_penalty++; + + ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); + + const int kMinimalPriority = get_minimal_priority(thread); + if (thread->priority - oldPenalty <= kMinimalPriority) { + int32 core = schedulerThreadData->previous_core; + ASSERT(core >= 0); + + int32 additionalPenalty = schedulerThreadData->additional_penalty; + if (additionalPenalty == 0 && !sSingleCore) { + sCPUBoundThreads++; + sCoreEntries[core].fCPUBoundThreads++; + + update_thread_heaps(core); + } + + const int kSmallTaskThreshold = 50; + if (additionalPenalty > kSmallTaskThreshold && !sSingleCore) { + if (sSmallTaskCore == core) + disable_small_task_packing(); + } + + schedulerThreadData->priority_penalty = oldPenalty; + schedulerThreadData->additional_penalty++; + } +} + + +static inline void +cancel_penalty(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + if (schedulerThreadData->priority_penalty != 0) + TRACE("cancelling thread %ld penalty\n", thread->id); + + switch (sSchedulerMode) { + case SCHEDULER_MODE_PERFORMANCE: + schedulerThreadData->additional_penalty = 0; + schedulerThreadData->priority_penalty = 0; + break; + + case SCHEDULER_MODE_POWER_SAVING: + if (schedulerThreadData->additional_penalty != 0) + schedulerThreadData->additional_penalty /= 2; + else if (schedulerThreadData->priority_penalty != 0) + schedulerThreadData->priority_penalty--; + break; + + default: + break; + } +} + + +static inline void +update_priority_heaps(int32 cpu, int32 priority) +{ + int32 core = sCPUToCore[cpu]; + + sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); + + if (sSingleCore) + return; + + int32 maxPriority + = CPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); + int32 corePriority = CorePriorityHeap::GetKey(&sCoreEntries[core]); + + if (corePriority != maxPriority) { + if (maxPriority == B_IDLE_PRIORITY) { + sCorePriorityHeap->ModifyKey(&sCoreEntries[core], B_IDLE_PRIORITY); + ASSERT(sCorePriorityHeap->PeekRoot() == &sCoreEntries[core]); + sCorePriorityHeap->RemoveRoot(); + } else if (corePriority == B_IDLE_PRIORITY) + sCorePriorityHeap->Insert(&sCoreEntries[core], maxPriority); + else + sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); + + int32 package = sCPUToPackage[cpu]; + PackageEntry* packageEntry = &sPackageEntries[package]; + if (maxPriority == B_IDLE_PRIORITY) { + // core goes idle + ASSERT(packageEntry->fIdleCoreCount >= 0); + ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount++; + packageEntry->fIdleCores.Add(&sCoreEntries[core]); + + if (packageEntry->fIdleCoreCount == 1) { + // first core on that package to go idle + + if (packageEntry->fCoreCount > 1) + sPackageUsageHeap->Insert(packageEntry, 1); + else + sIdlePackageList->Add(packageEntry); + } else if (packageEntry->fIdleCoreCount + == packageEntry->fCoreCount) { + // package goes idle + sPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); + sPackageUsageHeap->RemoveMinimum(); + + sIdlePackageList->Add(packageEntry); + } else { + sPackageUsageHeap->ModifyKey(packageEntry, + packageEntry->fIdleCoreCount); + } + } else if (corePriority == B_IDLE_PRIORITY) { + // core wakes up + ASSERT(packageEntry->fIdleCoreCount > 0); + ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount--; + packageEntry->fIdleCores.Remove(&sCoreEntries[core]); + + if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { + // package wakes up + sIdlePackageList->Remove(packageEntry); + + if (packageEntry->fIdleCoreCount > 0) { + sPackageUsageHeap->Insert(packageEntry, + packageEntry->fIdleCoreCount); + } + } else if (packageEntry->fIdleCoreCount == 0) { + // no more idle cores in the package + sPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); + sPackageUsageHeap->RemoveMinimum(); + } else { + sPackageUsageHeap->ModifyKey(packageEntry, + packageEntry->fIdleCoreCount); + } + } + } +} + + +static int32 +choose_core_performance(Thread* thread) +{ + CoreEntry* entry; + + if (sIdlePackageList->Last() != NULL) { + // wake new package + PackageEntry* package = sIdlePackageList->Last(); + entry = package->fIdleCores.Last(); + } else if (sPackageUsageHeap->PeekMaximum() != NULL) { + // wake new core + PackageEntry* package = sPackageUsageHeap->PeekMaximum(); + entry = package->fIdleCores.Last(); + } else { + // no idle cores, use least occupied core + entry = sCorePriorityHeap->PeekRoot(); + + int32 priority = get_effective_priority(thread); + if (CorePriorityHeap::GetKey(entry) >= priority) { + entry = sCoreThreadHeap->PeekMinimum(); + if (entry == NULL) + entry = sCoreCPUBoundThreadHeap->PeekMinimum(); + } + } + + ASSERT(entry != NULL); + return entry->fCoreID; +} + + +static inline bool +is_task_small(Thread* thread) +{ + int32 priority = get_effective_priority(thread); + int32 penalty = thread->scheduler_data->priority_penalty; + return penalty == 0 || priority >= B_DISPLAY_PRIORITY; +} + + +static int32 +choose_core_power_saving(Thread* thread) +{ + CoreEntry* entry; + + int32 priority = get_effective_priority(thread); + + if (sDisableSmallTaskPacking > 0) + sDisableSmallTaskPacking--; + + if (!sDisableSmallTaskPacking && is_task_small(thread) + && sCoreThreadHeap->PeekMaximum() != NULL) { + // try to pack all threads on one core + if (sSmallTaskCore < 0) + sSmallTaskCore = sCoreThreadHeap->PeekMaximum()->fCoreID; + entry = &sCoreEntries[sSmallTaskCore]; + } else if (sCorePriorityHeap->PeekRoot() != NULL + && CorePriorityHeap::GetKey(sCorePriorityHeap->PeekRoot()) + < priority) { + // run immediately on already woken core + entry = sCorePriorityHeap->PeekRoot(); + } else if (sPackageUsageHeap->PeekMinimum() != NULL) { + // wake new core + PackageEntry* package = sPackageUsageHeap->PeekMinimum(); + entry = package->fIdleCores.Last(); + } else if (sIdlePackageList->Last() != NULL) { + // wake new package + PackageEntry* package = sIdlePackageList->Last(); + entry = package->fIdleCores.Last(); + } else { + // no idle cores, use least occupied core + entry = sCoreThreadHeap->PeekMinimum(); + if (entry == NULL) + entry = sCoreCPUBoundThreadHeap->PeekMinimum(); + } + + ASSERT(entry != NULL); + return entry->fCoreID; +} + + +static inline int32 +choose_core(Thread* thread) +{ + ASSERT(!sSingleCore); + return sChooseCore(thread); +} + + +static inline int32 +choose_cpu(int32 core) +{ + CPUEntry* entry = sCPUPriorityHeaps[core].PeekMinimum(); + ASSERT(entry != NULL); + return entry->fCPUNumber; +} + + +static bool +should_rebalance(Thread* thread) +{ + ASSERT(!sSingleCore); + + if (thread_is_idle_thread(thread)) + return false; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; + + // If this is a cpu bound thread and we have significantly more such threads + // than the average get rid of this one. + if (schedulerThreadData->additional_penalty != 0) { + int32 averageCPUBound = sCPUBoundThreads / sRunQueueCount; + return coreEntry->fCPUBoundThreads - averageCPUBound > 1; + } + + // If this thread is not cpu bound but we have at least one consider giving + // this one to someone less busy. + int32 averageThread = sAssignedThreads / sRunQueueCount; + if (coreEntry->fCPUBoundThreads > 0) { + CoreEntry* other = sCoreThreadHeap->PeekMinimum(); + if (other != NULL + && CoreThreadHeap::GetKey(other) <= averageThread) { + return true; + } + } + + int32 threadsAboveAverage = coreEntry->fThreads - averageThread; + + // All cores try to give us small tasks, check whether we have enough. + const int kSmallTaskCountThreshold = 5; + if (sDisableSmallTaskPacking == 0 && sSmallTaskCore == coreEntry->fCoreID) { + if (threadsAboveAverage > kSmallTaskCountThreshold) { + if (!is_task_small(thread)) + return true; + } else if (threadsAboveAverage > 2 * kSmallTaskCountThreshold) { + disable_small_task_packing(); + } + } + + // Try our luck at small task packing. + if (sDisableSmallTaskPacking == 0 && is_task_small(thread)) + return sSmallTaskCore != coreEntry->fCoreID; + + // No cpu bound threads - the situation is quite good. Make sure it + // won't get much worse... + const int32 kBalanceThreshold = 3; + return threadsAboveAverage > kBalanceThreshold; +} + + +static void +assign_active_thread_to_core(Thread* thread) +{ + if (thread_is_idle_thread(thread) || sSingleCore) + return; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + ASSERT(schedulerThreadData->previous_core >= 0); + int32 core = schedulerThreadData->previous_core; + + sCoreEntries[core].fThreads++; + sAssignedThreads++; + + if (schedulerThreadData->additional_penalty != 0) { + sCoreEntries[core].fCPUBoundThreads++; + sCPUBoundThreads++; + } + + update_thread_heaps(core); +} + + +static inline void +thread_goes_away(Thread* thread) +{ + if (thread_is_idle_thread(thread)) + return; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + ASSERT(schedulerThreadData->previous_core >= 0); + int32 core = schedulerThreadData->previous_core; + + schedulerThreadData->went_sleep = system_time(); + schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; + + if (sSingleCore) + return; + + ASSERT(sCoreEntries[core].fThreads > 0); + ASSERT(sCoreEntries[core].fThreads > sCoreEntries[core].fCPUBoundThreads + || (sCoreEntries[core].fThreads == sCoreEntries[core].fCPUBoundThreads + && schedulerThreadData->additional_penalty != 0)); + sCoreEntries[core].fThreads--; + sAssignedThreads--; + + if (schedulerThreadData->additional_penalty != 0) { + ASSERT(sCoreEntries[core].fCPUBoundThreads > 0); + sCoreEntries[core].fCPUBoundThreads--; + sCPUBoundThreads--; + } + + update_thread_heaps(core); +} + + +static void +enqueue(Thread* thread, bool newOne) +{ + ASSERT(thread != NULL); + + thread->state = thread->next_state = B_THREAD_READY; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; + if (newOne && hasSlept > kThreadQuantum) + cancel_penalty(thread); + + int32 threadPriority = get_effective_priority(thread); + + T(EnqueueThread(thread, threadPriority)); + + bool pinned = thread->pinned_to_cpu > 0; + int32 targetCPU = -1; + int32 targetCore; + if (pinned) { + targetCPU = thread->previous_cpu->cpu_num; + targetCore = sCPUToCore[targetCPU]; + ASSERT(targetCore == schedulerThreadData->previous_core); + + if (newOne) + assign_active_thread_to_core(thread); + } else if (sSingleCore) { + targetCore = 0; + targetCPU = choose_cpu(targetCore); + + schedulerThreadData->previous_core = targetCore; + } else if (schedulerThreadData->previous_core < 0 + || (newOne && has_cache_expired(thread)) + || should_rebalance(thread)) { + + if (thread_is_idle_thread(thread)) { + targetCPU = thread->previous_cpu->cpu_num; + targetCore = sCPUToCore[targetCPU]; + } else { + if (!newOne) + thread_goes_away(thread); + + targetCore = choose_core(thread); + targetCPU = choose_cpu(targetCore); + } + + schedulerThreadData->previous_core = targetCore; + assign_active_thread_to_core(thread); + } else { + targetCore = schedulerThreadData->previous_core; + targetCPU = choose_cpu(targetCore); + if (newOne) + assign_active_thread_to_core(thread); + } + + ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); + ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); + + TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, + threadPriority, targetCore); + if (pinned) + sPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); + else + sRunQueues[targetCore].PushBack(thread, threadPriority); + + schedulerThreadData->cpu_bound = true; + schedulerThreadData->time_left = 0; + schedulerThreadData->stolen_time = 0; + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, + thread); + + Thread* targetThread = gCPU[targetCPU].running_thread; + int32 targetPriority = get_effective_priority(targetThread); + + TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, + targetPriority); + + if (threadPriority > targetPriority) { + targetThread->scheduler_data->lost_cpu = true; + + // It is possible that another CPU schedules the thread before the + // target CPU. However, since the target CPU is sent an ICI it will + // reschedule anyway and update its heap key to the correct value. + update_priority_heaps(targetCPU, threadPriority); + + if (targetCPU == smp_get_current_cpu()) + gCPU[targetCPU].invoke_scheduler = true; + else { + smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, + SMP_MSG_FLAG_ASYNC); + } + } +} + + +/*! Enqueues the thread into the run queue. + Note: thread lock must be held when entering this function +*/ +void +scheduler_enqueue_in_run_queue(Thread *thread) +{ + TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, + thread->priority); + enqueue(thread, true); +} + + +static inline void +put_back(Thread* thread) +{ + bool pinned = sPinnedRunQueues != NULL && thread->pinned_to_cpu > 0; + + if (pinned) { + int32 pinnedCPU = thread->previous_cpu->cpu_num; + sPinnedRunQueues[pinnedCPU].PushFront(thread, + get_effective_priority(thread)); + } else { + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); + sRunQueues[previousCore].PushFront(thread, + get_effective_priority(thread)); + } +} + + +/*! Sets the priority of a thread. + Note: thread lock must be held when entering this function +*/ +void +scheduler_set_thread_priority(Thread *thread, int32 priority) +{ + if (priority == thread->priority) + return; + + TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", + thread->id, priority, thread->priority, + get_effective_priority(thread)); + + if (thread->state == B_THREAD_RUNNING) + thread_goes_away(thread); + + if (thread->state != B_THREAD_READY) { + cancel_penalty(thread); + thread->priority = priority; + + if (thread->state == B_THREAD_RUNNING) { + assign_active_thread_to_core(thread); + update_priority_heaps(thread->cpu->cpu_num, priority); + } + return; + } + + // The thread is in the run queue. We need to remove it and re-insert it at + // a new position. + + T(RemoveThread(thread)); + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, + thread); + + // remove thread from run queue + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); + sRunQueues[previousCore].Remove(thread); + thread_goes_away(thread); + + // set priority and re-insert + cancel_penalty(thread); + thread->priority = priority; + + scheduler_enqueue_in_run_queue(thread); +} + + +static int32 +reschedule_event(timer *unused) +{ + // This function is called as a result of the timer event set by the + // scheduler. Make sure the reschedule() is invoked. + Thread* thread= thread_get_current_thread(); + + thread->scheduler_data->lost_cpu = true; + thread->cpu->invoke_scheduler = true; + thread->cpu->preempted = 1; + return B_HANDLED_INTERRUPT; +} + + +static inline bool +quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + if (hasYielded) { + schedulerThreadData->time_left = 0; + return true; + } + + bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; + schedulerThreadData->time_left -= time_used; + schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); + + // too little time left, it's better make the next quantum a bit longer + if (wasPreempted || schedulerThreadData->time_left <= kThreadQuantum / 50) { + schedulerThreadData->stolen_time += schedulerThreadData->time_left; + schedulerThreadData->time_left = 0; + } + + return schedulerThreadData->time_left == 0; +} + + +static inline bigtime_t +quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, + int32 maxPriority, int32 minPriority, int32 priority) +{ + ASSERT(priority <= maxPriority); + ASSERT(priority >= minPriority); + + bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); + result /= maxPriority - minPriority; + return maxQuantum - result; +} + + +static inline bigtime_t +get_base_quantum(Thread* thread) +{ + int32 priority = get_effective_priority(thread); + + if (priority >= B_URGENT_DISPLAY_PRIORITY) + return kThreadQuantum; + if (priority > B_NORMAL_PRIORITY) { + return quantum_linear_interpolation(kThreadQuantum * 4, + kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, + priority); + } + return quantum_linear_interpolation(kThreadQuantum * 64, + kThreadQuantum * 4, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); +} + + +static inline bigtime_t +compute_quantum(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + bigtime_t quantum; + if (schedulerThreadData->time_left != 0) + quantum = schedulerThreadData->time_left; + else + quantum = get_base_quantum(thread); + + quantum += schedulerThreadData->stolen_time; + schedulerThreadData->stolen_time = 0; + + schedulerThreadData->time_left = quantum; + schedulerThreadData->quantum_start = system_time(); + + return quantum; +} + + +static inline Thread* +dequeue_thread(int32 thisCPU) +{ + int32 thisCore = sCPUToCore[thisCPU]; + Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); + + Thread* pinnedThread = NULL; + if (sPinnedRunQueues != NULL) + pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); + + if (sharedThread == NULL && pinnedThread == NULL) + return NULL; + + int32 pinnedPriority = -1; + if (pinnedThread != NULL) + pinnedPriority = get_effective_priority(pinnedThread); + + int32 sharedPriority = -1; + if (sharedThread != NULL) + sharedPriority = get_effective_priority(sharedThread); + + if (sharedPriority > pinnedPriority) { + sRunQueues[thisCore].Remove(sharedThread); + return sharedThread; + } + + sPinnedRunQueues[thisCPU].Remove(pinnedThread); + return pinnedThread; +} + + +static inline void +track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) +{ + if (!thread_is_idle_thread(oldThread)) { + bigtime_t active + = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) + + (oldThread->user_time - oldThread->cpu->last_user_time); + + atomic_add64(&oldThread->cpu->active_time, active); + sCoreEntries[thisCore].fActiveTime += active; + } + + if (!thread_is_idle_thread(nextThread)) { + oldThread->cpu->last_kernel_time = nextThread->kernel_time; + oldThread->cpu->last_user_time = nextThread->user_time; + } +} + + +static void +_scheduler_reschedule(void) +{ + Thread* oldThread = thread_get_current_thread(); + + int32 thisCPU = smp_get_current_cpu(); + int32 thisCore = sCPUToCore[thisCPU]; + + TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, + oldThread->id); + + oldThread->state = oldThread->next_state; + scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; + + // update CPU heap so that old thread would have CPU properly chosen + Thread* nextThread = sRunQueues[thisCore].PeekMaximum(); + if (nextThread != NULL) { + update_priority_heaps(thisCPU, + get_effective_priority(nextThread)); + } + + switch (oldThread->next_state) { + case B_THREAD_RUNNING: + case B_THREAD_READY: + if (!schedulerOldThreadData->lost_cpu) + schedulerOldThreadData->cpu_bound = false; + + if (quantum_ended(oldThread, oldThread->cpu->preempted, + oldThread->has_yielded)) { + if (schedulerOldThreadData->cpu_bound) + increase_penalty(oldThread); + + TRACE("enqueueing thread %ld into run queue priority = %ld\n", + oldThread->id, get_effective_priority(oldThread)); + enqueue(oldThread, false); + } else { + TRACE("putting thread %ld back in run queue priority = %ld\n", + oldThread->id, get_effective_priority(oldThread)); + put_back(oldThread); + } + + break; + case B_THREAD_SUSPENDED: + thread_goes_away(oldThread); + TRACE("reschedule(): suspending thread %ld\n", oldThread->id); + break; + case THREAD_STATE_FREE_ON_RESCHED: + thread_goes_away(oldThread); + break; + default: + thread_goes_away(oldThread); + TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", + oldThread->id, oldThread->next_state); + break; + } + + oldThread->has_yielded = false; + schedulerOldThreadData->lost_cpu = false; + + // select thread with the biggest priority + if (oldThread->cpu->disabled) { + ASSERT(sPinnedRunQueues != NULL); + nextThread = sPinnedRunQueues[thisCPU].PeekMaximum(); + if (nextThread != NULL) + sPinnedRunQueues[thisCPU].Remove(nextThread); + else { + nextThread = sRunQueues[thisCore].GetHead(B_IDLE_PRIORITY); + if (nextThread != NULL) + sRunQueues[thisCore].Remove(nextThread); + } + } else + nextThread = dequeue_thread(thisCPU); + if (!nextThread) + panic("reschedule(): run queues are empty!\n"); + + TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, + nextThread->id); + + T(ScheduleThread(nextThread, oldThread)); + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, + oldThread, nextThread); + + // update CPU heap + update_priority_heaps(thisCPU, + get_effective_priority(nextThread)); + + nextThread->state = B_THREAD_RUNNING; + nextThread->next_state = B_THREAD_READY; + ASSERT(nextThread->scheduler_data->previous_core == thisCore); + //nextThread->scheduler_data->previous_core = thisCore; + + // track kernel time (user time is tracked in thread_at_kernel_entry()) + scheduler_update_thread_times(oldThread, nextThread); + + // track CPU activity + track_cpu_activity(oldThread, nextThread, thisCore); + + if (nextThread != oldThread || oldThread->cpu->preempted) { + timer* quantumTimer = &oldThread->cpu->quantum_timer; + if (!oldThread->cpu->preempted) + cancel_timer(quantumTimer); + + oldThread->cpu->preempted = 0; + if (!thread_is_idle_thread(nextThread)) { + bigtime_t quantum = compute_quantum(oldThread); + add_timer(quantumTimer, &reschedule_event, quantum, + B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); + } + + if (nextThread != oldThread) + scheduler_switch_thread(oldThread, nextThread); + } +} + + +/*! Runs the scheduler. + Note: expects thread spinlock to be held +*/ +void +scheduler_reschedule(void) +{ + if (!sSchedulerEnabled) { + Thread* thread = thread_get_current_thread(); + if (thread != NULL && thread->next_state != B_THREAD_READY) + panic("scheduler_reschedule_no_op() called in non-ready thread"); + return; + } + + _scheduler_reschedule(); +} + + +status_t +scheduler_on_thread_create(Thread* thread, bool idleThread) +{ + thread->scheduler_data = new (std::nothrow)scheduler_thread_data; + if (thread->scheduler_data == NULL) + return B_NO_MEMORY; + return B_OK; +} + + +void +scheduler_on_thread_init(Thread* thread) +{ + thread->scheduler_data->Init(); +} + + +void +scheduler_on_thread_destroy(Thread* thread) +{ + delete thread->scheduler_data; +} + + +/*! This starts the scheduler. Must be run in the context of the initial idle + thread. Interrupts must be disabled and will be disabled when returning. +*/ +void +scheduler_start(void) +{ + SpinLocker schedulerLocker(gSchedulerLock); + + _scheduler_reschedule(); +} + + +static status_t +set_operation_mode(scheduler_mode mode) +{ + if (mode != SCHEDULER_MODE_PERFORMANCE + && mode != SCHEDULER_MODE_POWER_SAVING) { + return B_BAD_VALUE; + } + +#ifdef TRACE_SCHEDULER + const char* modeNames = { "performance", "power saving" }; +#endif + TRACE("switching scheduler to %s mode\n", modeNames[mode]); + + sSchedulerMode = mode; + switch (mode) { + case SCHEDULER_MODE_PERFORMANCE: + sDisableSmallTaskPacking = -1; + sSmallTaskCore = -1; + sChooseCore = choose_core_performance; + break; + + case SCHEDULER_MODE_POWER_SAVING: + sDisableSmallTaskPacking = 0; + sSmallTaskCore = -1; + sChooseCore = choose_core_power_saving; + break; + + default: + break; + } + + return B_OK; +} + + +static void +traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) +{ + switch (node->level) { + case CPU_TOPOLOGY_SMT: + sCPUToCore[node->id] = coreID; + sCPUToPackage[node->id] = packageID; + return; + + case CPU_TOPOLOGY_CORE: + coreID = node->id; + break; + + case CPU_TOPOLOGY_PACKAGE: + packageID = node->id; + break; + + default: + break; + } + + for (int32 i = 0; i < node->children_count; i++) + traverse_topology_tree(node->children[i], packageID, coreID); +} + + +static status_t +build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) +{ + cpuCount = smp_get_num_cpus(); + + sCPUToCore = new(std::nothrow) int32[cpuCount]; + if (sCPUToCore == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuToCoreDeleter(sCPUToCore); + + sCPUToPackage = new(std::nothrow) int32[cpuCount]; + if (sCPUToPackage == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuToPackageDeleter(sCPUToPackage); + + coreCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) + coreCount++; + } + + packageCount = 0; + for (int32 i = 0; i < cpuCount; i++) { + if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 + && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { + packageCount++; + } + } + + cpu_topology_node* root = get_cpu_topology(); + traverse_topology_tree(root, 0, 0); + + cpuToCoreDeleter.Detach(); + cpuToPackageDeleter.Detach(); + return B_OK; +} + + +static status_t +_scheduler_init() +{ + // create logical processor to core and package mappings + int32 cpuCount, coreCount, packageCount; + status_t result = build_topology_mappings(cpuCount, coreCount, + packageCount); + if (result != B_OK) + return result; + sRunQueueCount = coreCount; + sSingleCore = coreCount == 1; + + // create package heap and idle package stack + sPackageEntries = new(std::nothrow) PackageEntry[packageCount]; + if (sPackageEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter packageEntriesDeleter(sPackageEntries); + + sPackageUsageHeap = new(std::nothrow) PackageHeap(packageCount); + if (sPackageUsageHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter packageHeapDeleter(sPackageUsageHeap); + + sIdlePackageList = new(std::nothrow) IdlePackageList; + if (sIdlePackageList == NULL) + return B_NO_MEMORY; + ObjectDeleter packageListDeleter(sIdlePackageList); + + for (int32 i = 0; i < packageCount; i++) { + sPackageEntries[i].fPackageID = i; + sPackageEntries[i].fIdleCoreCount = coreCount / packageCount; + sPackageEntries[i].fCoreCount = coreCount / packageCount; + sIdlePackageList->Insert(&sPackageEntries[i]); + } + + // create logical processor and core heaps + sCPUEntries = new CPUEntry[cpuCount]; + if (sCPUEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuEntriesDeleter(sCPUEntries); + + sCoreEntries = new CoreEntry[coreCount]; + if (sCoreEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter coreEntriesDeleter(sCoreEntries); + + sCorePriorityHeap = new CorePriorityHeap(coreCount); + if (sCorePriorityHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter corePriorityHeapDeleter(sCorePriorityHeap); + + sCoreThreadHeap = new CoreThreadHeap; + if (sCoreThreadHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter coreThreadHeapDeleter(sCoreThreadHeap); + + sCoreCPUBoundThreadHeap = new CoreThreadHeap(coreCount); + if (sCoreCPUBoundThreadHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter coreCPUThreadHeapDeleter( + sCoreCPUBoundThreadHeap); + + for (int32 i = 0; i < coreCount; i++) { + sCoreEntries[i].fCoreID = i; + sCoreEntries[i].fActiveTime = 0; + sCoreEntries[i].fThreads = 0; + sCoreEntries[i].fCPUBoundThreads = 0; + + status_t result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); + if (result != B_OK) + return result; + + result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); + if (result != B_OK) + return result; + sCorePriorityHeap->RemoveRoot(); + } + + sCPUPriorityHeaps = new CPUHeap[coreCount]; + if (sCPUPriorityHeaps == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); + + for (int32 i = 0; i < cpuCount; i++) { + sCPUEntries[i].fCPUNumber = i; + int32 core = sCPUToCore[i]; + + int32 package = sCPUToPackage[i]; + if (sCPUPriorityHeaps[core].PeekMaximum() == NULL) + sPackageEntries[package].fIdleCores.Insert(&sCoreEntries[core]); + + status_t result + = sCPUPriorityHeaps[core].Insert(&sCPUEntries[i], B_IDLE_PRIORITY); + if (result != B_OK) + return result; + } + + // create per-logical processor run queues for pinned threads + TRACE("scheduler_init(): creating %" B_PRId32 " per-cpu queue%s\n", + cpuCount, cpuCount != 1 ? "s" : ""); + + sPinnedRunQueues = new(std::nothrow) ThreadRunQueue[cpuCount]; + if (sPinnedRunQueues == NULL) + return B_NO_MEMORY; + ArrayDeleter pinnedRunQueuesDeleter(sPinnedRunQueues); + for (int i = 0; i < cpuCount; i++) { + status_t result = sPinnedRunQueues[i].GetInitStatus(); + if (result != B_OK) + return result; + } + + // create per-core run queues + TRACE("scheduler_init(): creating %" B_PRId32 " per-core queue%s\n", + coreCount, coreCount != 1 ? "s" : ""); + + sRunQueues = new(std::nothrow) ThreadRunQueue[coreCount]; + if (sRunQueues == NULL) + return B_NO_MEMORY; + ArrayDeleter runQueuesDeleter(sRunQueues); + for (int i = 0; i < coreCount; i++) { + status_t result = sRunQueues[i].GetInitStatus(); + if (result != B_OK) + return result; + } + +#if 1 + set_operation_mode(SCHEDULER_MODE_POWER_SAVING); +#else + set_operation_mode(SCHEDULER_MODE_PERFORMANCE); +#endif + + add_debugger_command_etc("run_queue", &dump_run_queue, + "List threads in run queue", "\nLists threads in run queue", 0); + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", + 0); + if (!sSingleCore) { + add_debugger_command_etc("idle_cores", &dump_idle_cores, + "List idle cores", "\nList idle cores", 0); + } + + runQueuesDeleter.Detach(); + pinnedRunQueuesDeleter.Detach(); + coreCPUThreadHeapDeleter.Detach(); + coreThreadHeapDeleter.Detach(); + corePriorityHeapDeleter.Detach(); + cpuPriorityHeapDeleter.Detach(); + coreEntriesDeleter.Detach(); + cpuEntriesDeleter.Detach(); + packageEntriesDeleter.Detach(); + packageHeapDeleter.Detach(); + packageListDeleter.Detach(); + return B_OK; +} + + +void +scheduler_init(void) +{ + int32 cpuCount = smp_get_num_cpus(); + dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s and %" B_PRId32 + " cache level%s\n", cpuCount, cpuCount != 1 ? "s" : "", + gCPUCacheLevelCount, gCPUCacheLevelCount != 1 ? "s" : ""); + + status_t result = _scheduler_init(); + if (result != B_OK) + panic("scheduler_init: failed to initialize scheduler\n"); + +#if SCHEDULER_TRACING + add_debugger_command_etc("scheduler", &cmd_scheduler, + "Analyze scheduler tracing information", + "\n" + "Analyzes scheduler tracing information for a given thread.\n" + " - ID of the thread.\n", 0); +#endif +} + + +void +scheduler_enable_scheduling(void) +{ + sSchedulerEnabled = true; } @@ -59,65 +1715,6 @@ scheduler_remove_listener(struct SchedulerListener* listener) } -static bool -should_use_affine_scheduler(int32 cpuCount) -{ - if (cpuCount < 2) - return false; - - for (int32 i = 1; i < cpuCount; i++) { - for (int32 j = 0; j < gCPUCacheLevelCount; j++) { - if (gCPU[i].cache_id[j] != gCPU[i - 1].cache_id[j]) - return true; - } - } - - return false; -} - - -void -scheduler_init(void) -{ - int32 cpuCount = smp_get_num_cpus(); - dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s and %" B_PRId32 - " cache level%s\n", cpuCount, cpuCount != 1 ? "s" : "", - gCPUCacheLevelCount, gCPUCacheLevelCount != 1 ? "s" : ""); - - status_t result; - if (should_use_affine_scheduler(cpuCount)) { - dprintf("scheduler_init: using affine scheduler\n"); - result = scheduler_affine_init(); - } else { - dprintf("scheduler_init: using simple scheduler\n"); - result = scheduler_simple_init(); - } - - if (result != B_OK) - panic("scheduler_init: failed to initialize scheduler\n"); - - // Disable rescheduling until the basic kernel initialization is done and - // CPUs are ready to enable interrupts. - sRescheduleFunction = gScheduler->reschedule; - gScheduler->reschedule = scheduler_reschedule_no_op; - -#if SCHEDULER_TRACING - add_debugger_command_etc("scheduler", &cmd_scheduler, - "Analyze scheduler tracing information", - "\n" - "Analyzes scheduler tracing information for a given thread.\n" - " - ID of the thread.\n", 0); -#endif -} - - -void -scheduler_enable_scheduling(void) -{ - gScheduler->reschedule = sRescheduleFunction; -} - - // #pragma mark - Syscalls @@ -138,7 +1735,16 @@ _user_estimate_max_scheduling_latency(thread_id id) } BReference threadReference(thread, true); - // ask the scheduler for the thread's latency - InterruptsSpinLocker locker(gSchedulerLock); - return gScheduler->estimate_max_scheduling_latency(thread); + // TODO: This is probably meant to be called periodically to return the + // current estimate depending on the system usage; we return fixed estimates + // per thread priority, though. + + if (thread->priority >= B_REAL_TIME_DISPLAY_PRIORITY) + return kMinThreadQuantum / 4; + if (thread->priority >= B_DISPLAY_PRIORITY) + return kMinThreadQuantum; + if (thread->priority < B_NORMAL_PRIORITY) + return 2 * kMaxThreadQuantum; + + return 2 * kMinThreadQuantum; } diff --git a/src/system/kernel/scheduler/scheduler_affine.cpp b/src/system/kernel/scheduler/scheduler_affine.cpp deleted file mode 100644 index a0fb24e109..0000000000 --- a/src/system/kernel/scheduler/scheduler_affine.cpp +++ /dev/null @@ -1,1676 +0,0 @@ -/* - * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. - * Copyright 2009, Rene Gollent, rene@gollent.com. - * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. - * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. - * Copyright 2002, Angelo Mottola, a.mottola@libero.it. - * Distributed under the terms of the MIT License. - * - * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. - * Distributed under the terms of the NewOS License. - */ - - -/*! The thread scheduler */ - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "RunQueue.h" -#include "scheduler_common.h" -#include "scheduler_tracing.h" - - -//#define TRACE_SCHEDULER -#ifdef TRACE_SCHEDULER -# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) -#else -# define TRACE(...) do { } while (false) -#endif - - -const bigtime_t kThreadQuantum = 1000; -const bigtime_t kMinThreadQuantum = 3000; -const bigtime_t kMaxThreadQuantum = 10000; - -const bigtime_t kCacheExpire = 100000; - -static int sDisableSmallTaskPacking; -static int32 sSmallTaskCore; - -static bool sSingleCore; - -static scheduler_mode sSchedulerMode; - -static int32 (*sChooseCore)(Thread* thread); - - -// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical -// processors in the heap belong to. Since there are no cache affinity issues -// at this level and the run queue is shared among all logical processors on -// the core the only real concern is to make lower priority threads give way to -// the higher priority threads. -struct CPUEntry : public MinMaxHeapLinkImpl { - int32 fCPUNumber; -}; -typedef MinMaxHeap AffineCPUHeap; -static CPUEntry* sCPUEntries; -static AffineCPUHeap* sCPUPriorityHeaps; - -struct CoreEntry : public DoublyLinkedListLinkImpl { - HeapLink fPriorityHeapLink; - MinMaxHeapLink fThreadHeapLink; - - int32 fCoreID; - - bigtime_t fActiveTime; - - int32 fCPUBoundThreads; - int32 fThreads; -}; - -static CoreEntry* sCoreEntries; -typedef Heap, - HeapMemberGetLink > - AffineCorePriorityHeap; -static AffineCorePriorityHeap* sCorePriorityHeap; - -typedef MinMaxHeap, - MinMaxHeapMemberGetLink > - AffineCoreThreadHeap; -static AffineCoreThreadHeap* sCoreThreadHeap; -static AffineCoreThreadHeap* sCoreCPUBoundThreadHeap; - -static int32 sCPUBoundThreads; -static int32 sAssignedThreads; - -// sPackageUsageHeap is used to decide which core should be woken up from the -// idle state. When aiming for performance we should use as many packages as -// possible with as little cores active in each package as possible (so that the -// package can enter any boost mode if it has one and the active core have more -// of the shared cache for themselves. If power saving is the main priority we -// should keep active cores on as little packages as possible (so that other -// packages can go to the deep state of sleep). The heap stores only packages -// with at least one core active and one core idle. The packages with all cores -// idle are stored in sPackageIdleList (in LIFO manner). -struct PackageEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { - int32 fPackageID; - - DoublyLinkedList fIdleCores; - int32 fIdleCoreCount; - - int32 fCoreCount; -}; -typedef MinMaxHeap AffinePackageHeap; -typedef DoublyLinkedList AffineIdlePackageList; - -static PackageEntry* sPackageEntries; -static AffinePackageHeap* sPackageUsageHeap; -static AffineIdlePackageList* sIdlePackageList; - -// The run queues. Holds the threads ready to run ordered by priority. -// One queue per schedulable target per core. Additionally, each -// logical processor has its sPinnedRunQueues used for scheduling -// pinned threads. -typedef RunQueue AffineRunQueue; -static AffineRunQueue* sRunQueues; -static AffineRunQueue* sPinnedRunQueues; -static int32 sRunQueueCount; - -// Since CPU IDs used internally by the kernel bear no relation to the actual -// CPU topology the following arrays are used to efficiently get the core -// and the package that CPU in question belongs to. -static int32* sCPUToCore; -static int32* sCPUToPackage; - - -struct scheduler_thread_data { - scheduler_thread_data() { Init(); } - inline void Init(); - - int32 priority_penalty; - int32 additional_penalty; - - bool lost_cpu; - bool cpu_bound; - - bigtime_t time_left; - bigtime_t stolen_time; - bigtime_t quantum_start; - - bigtime_t went_sleep; - bigtime_t went_sleep_active; - - int32 previous_core; -}; - - -void -scheduler_thread_data::Init() -{ - priority_penalty = 0; - additional_penalty = 0; - - time_left = 0; - stolen_time = 0; - - went_sleep = 0; - went_sleep_active = 0; - - lost_cpu = false; - cpu_bound = true; - - previous_core = -1; -} - - -static inline int -affine_get_minimal_priority(Thread* thread) -{ - return min_c(thread->priority, 25) / 5; -} - - -static inline int32 -affine_get_thread_penalty(Thread* thread) -{ - int32 penalty = thread->scheduler_data->priority_penalty; - - const int kMinimalPriority = affine_get_minimal_priority(thread); - if (kMinimalPriority > 0) { - penalty - += thread->scheduler_data->additional_penalty % kMinimalPriority; - } - - return penalty; -} - - -static inline int32 -affine_get_effective_priority(Thread* thread) -{ - if (thread->priority == B_IDLE_PRIORITY) - return thread->priority; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return thread->priority; - - int32 effectivePriority = thread->priority; - effectivePriority -= affine_get_thread_penalty(thread); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; -} - - -static void -dump_queue(AffineRunQueue::ConstIterator& iterator) -{ - if (!iterator.HasNext()) - kprintf("Run queue is empty.\n"); - else { - kprintf("thread id priority penalty name\n"); - while (iterator.HasNext()) { - Thread* thread = iterator.Next(); - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", - thread, thread->id, thread->priority, - affine_get_thread_penalty(thread), thread->name); - } - } -} - - -static int -dump_run_queue(int argc, char **argv) -{ - int32 cpuCount = smp_get_num_cpus(); - int32 coreCount = 0; - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) - sCPUToCore[i] = coreCount++; - } - - AffineRunQueue::ConstIterator iterator; - for (int32 i = 0; i < coreCount; i++) { - kprintf("\nCore %" B_PRId32 " run queue:\n", i); - iterator = sRunQueues[i].GetConstIterator(); - dump_queue(iterator); - } - - for (int32 i = 0; i < cpuCount; i++) { - iterator = sPinnedRunQueues[i].GetConstIterator(); - - if (iterator.HasNext()) { - kprintf("\nCPU %" B_PRId32 " run queue:\n", i); - dump_queue(iterator); - } - } - - return 0; -} - - -static void -dump_heap(AffineCPUHeap* heap) -{ - AffineCPUHeap temp(smp_get_num_cpus()); - - kprintf("cpu priority actual priority\n"); - CPUEntry* entry = heap->PeekMinimum(); - while (entry) { - int32 cpu = entry->fCPUNumber; - int32 key = AffineCPUHeap::GetKey(entry); - kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, key, - affine_get_effective_priority(gCPU[cpu].running_thread)); - - heap->RemoveMinimum(); - temp.Insert(entry, key); - - entry = heap->PeekMinimum(); - } - - entry = temp.PeekMinimum(); - while (entry) { - int32 key = AffineCPUHeap::GetKey(entry); - temp.RemoveMinimum(); - heap->Insert(entry, key); - entry = temp.PeekMinimum(); - } -} - - -static void -dump_core_thread_heap(AffineCoreThreadHeap* heap) -{ - AffineCoreThreadHeap temp(sRunQueueCount); - - CoreEntry* entry = heap->PeekMinimum(); - while (entry) { - int32 key = AffineCoreThreadHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 "\n", - entry->fCoreID, key, entry->fThreads, entry->fCPUBoundThreads); - - heap->RemoveMinimum(); - temp.Insert(entry, key); - - entry = heap->PeekMinimum(); - } - - entry = temp.PeekMinimum(); - while (entry) { - int32 key = AffineCoreThreadHeap::GetKey(entry); - temp.RemoveMinimum(); - heap->Insert(entry, key); - entry = temp.PeekMinimum(); - } -} - - -static int -dump_cpu_heap(int argc, char** argv) -{ - AffineCorePriorityHeap temp(sRunQueueCount); - - CoreEntry* entry = sCorePriorityHeap->PeekRoot(); - if (entry != NULL) - kprintf("core priority\n"); - else - kprintf("No active cores.\n"); - - while (entry) { - int32 core = entry->fCoreID; - int32 key = AffineCorePriorityHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %8" B_PRId32 "\n", core, key); - - sCorePriorityHeap->RemoveRoot(); - temp.Insert(entry, key); - - entry = sCorePriorityHeap->PeekRoot(); - } - - entry = temp.PeekRoot(); - while (entry) { - int32 key = AffineCorePriorityHeap::GetKey(entry); - temp.RemoveRoot(); - sCorePriorityHeap->Insert(entry, key); - entry = temp.PeekRoot(); - } - - kprintf("\ncore key threads cpu-bound\n"); - dump_core_thread_heap(sCoreThreadHeap); - dump_core_thread_heap(sCoreCPUBoundThreadHeap); - - for (int32 i = 0; i < sRunQueueCount; i++) { - kprintf("\nCore %" B_PRId32 " heap:\n", i); - dump_heap(&sCPUPriorityHeaps[i]); - } - - return 0; -} - - -static int -dump_idle_cores(int argc, char** argv) -{ - kprintf("Idle packages:\n"); - AffineIdlePackageList::ReverseIterator idleIterator - = sIdlePackageList->GetReverseIterator(); - - if (idleIterator.HasNext()) { - kprintf("package cores\n"); - - while (idleIterator.HasNext()) { - PackageEntry* entry = idleIterator.Next(); - kprintf("%-7" B_PRId32 " ", entry->fPackageID); - - DoublyLinkedList::ReverseIterator iterator - = entry->fIdleCores.GetReverseIterator(); - if (iterator.HasNext()) { - while (iterator.HasNext()) { - CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, - iterator.HasNext() ? ", " : ""); - } - } else - kprintf("-"); - kprintf("\n"); - } - } else - kprintf("No idle packages.\n"); - - AffinePackageHeap temp(smp_get_num_cpus()); - kprintf("\nPackages with idle cores:\n"); - - PackageEntry* entry = sPackageUsageHeap->PeekMinimum(); - if (entry == NULL) - kprintf("No packages.\n"); - else - kprintf("package count cores\n"); - - while (entry != NULL) { - kprintf("%-7" B_PRId32 " %-5" B_PRId32 " ", entry->fPackageID, - entry->fIdleCoreCount); - - DoublyLinkedList::ReverseIterator iterator - = entry->fIdleCores.GetReverseIterator(); - if (iterator.HasNext()) { - while (iterator.HasNext()) { - CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, - iterator.HasNext() ? ", " : ""); - } - } else - kprintf("-"); - kprintf("\n"); - - sPackageUsageHeap->RemoveMinimum(); - temp.Insert(entry, entry->fIdleCoreCount); - - entry = sPackageUsageHeap->PeekMinimum(); - } - - entry = temp.PeekMinimum(); - while (entry != NULL) { - int32 key = AffinePackageHeap::GetKey(entry); - temp.RemoveMinimum(); - sPackageUsageHeap->Insert(entry, key); - entry = temp.PeekMinimum(); - } - - return 0; -} - - -static inline bool -affine_has_cache_expired(Thread* thread) -{ - ASSERT(!sSingleCore); - - if (thread_is_idle_thread(thread)) - return false; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; - switch (sSchedulerMode) { - case SCHEDULER_MODE_PERFORMANCE: - return coreEntry->fActiveTime - - schedulerThreadData->went_sleep_active > kCacheExpire; - - case SCHEDULER_MODE_POWER_SAVING: - return system_time() - schedulerThreadData->went_sleep - > kCacheExpire; - - default: - return true; - } -} - - -static void -affine_dump_thread_data(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", - schedulerThreadData->priority_penalty); - - int32 additionalPenalty = 0; - const int kMinimalPriority = affine_get_minimal_priority(thread); - if (kMinimalPriority > 0) { - additionalPenalty - = schedulerThreadData->additional_penalty % kMinimalPriority; - } - kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", - additionalPenalty, schedulerThreadData->additional_penalty); - kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", - schedulerThreadData->stolen_time); - kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", - schedulerThreadData->went_sleep); - kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", - schedulerThreadData->went_sleep_active); - kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", - schedulerThreadData->previous_core); - if (schedulerThreadData->previous_core > 0 - && affine_has_cache_expired(thread)) { - kprintf("\tcache affinity has expired\n"); - } -} - - -static void -affine_update_thread_heaps(int32 core) -{ - ASSERT(!sSingleCore); - - CoreEntry* entry = &sCoreEntries[core]; - - ASSERT(entry->fCPUBoundThreads >= 0 - && entry->fCPUBoundThreads <= entry->fThreads); - ASSERT(entry->fThreads >= 0 - && entry->fThreads <= thread_max_threads()); - - int32 newKey = entry->fCPUBoundThreads * thread_max_threads(); - newKey += entry->fThreads; - - int32 oldKey = AffineCoreThreadHeap::GetKey(entry); - - if (oldKey == newKey) - return; - - if (newKey > thread_max_threads()) { - if (oldKey <= thread_max_threads()) { - sCoreThreadHeap->ModifyKey(entry, -1); - ASSERT(sCoreThreadHeap->PeekMinimum() == entry); - sCoreThreadHeap->RemoveMinimum(); - - sCoreCPUBoundThreadHeap->Insert(entry, newKey); - } else - sCoreCPUBoundThreadHeap->ModifyKey(entry, newKey); - } else { - if (oldKey > thread_max_threads()) { - sCoreCPUBoundThreadHeap->ModifyKey(entry, -1); - ASSERT(sCoreCPUBoundThreadHeap->PeekMinimum() == entry); - sCoreCPUBoundThreadHeap->RemoveMinimum(); - - sCoreThreadHeap->Insert(entry, newKey); - } else - sCoreThreadHeap->ModifyKey(entry, newKey); - } -} - - -static inline void -affine_disable_small_task_packing(void) -{ - ASSERT(!sSingleCore); - - ASSERT(sDisableSmallTaskPacking == 0); - ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); - - ASSERT(sAssignedThreads > 0); - sDisableSmallTaskPacking = sAssignedThreads * 64; - sSmallTaskCore = -1; -} - - -static inline void -affine_increase_penalty(Thread* thread) -{ - if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) - return; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return; - - TRACE("increasing thread %ld penalty\n", thread->id); - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - int32 oldPenalty = schedulerThreadData->priority_penalty++; - - ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - - const int kMinimalPriority = affine_get_minimal_priority(thread); - if (thread->priority - oldPenalty <= kMinimalPriority) { - int32 core = schedulerThreadData->previous_core; - ASSERT(core >= 0); - - int32 additionalPenalty = schedulerThreadData->additional_penalty; - if (additionalPenalty == 0 && !sSingleCore) { - sCPUBoundThreads++; - sCoreEntries[core].fCPUBoundThreads++; - - affine_update_thread_heaps(core); - } - - const int kSmallTaskThreshold = 50; - if (additionalPenalty > kSmallTaskThreshold && !sSingleCore) { - if (sSmallTaskCore == core) - affine_disable_small_task_packing(); - } - - schedulerThreadData->priority_penalty = oldPenalty; - schedulerThreadData->additional_penalty++; - } -} - - -static inline void -affine_cancel_penalty(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (schedulerThreadData->priority_penalty != 0) - TRACE("cancelling thread %ld penalty\n", thread->id); - - switch (sSchedulerMode) { - case SCHEDULER_MODE_PERFORMANCE: - schedulerThreadData->additional_penalty = 0; - schedulerThreadData->priority_penalty = 0; - break; - - case SCHEDULER_MODE_POWER_SAVING: - if (schedulerThreadData->additional_penalty != 0) - schedulerThreadData->additional_penalty /= 2; - else if (schedulerThreadData->priority_penalty != 0) - schedulerThreadData->priority_penalty--; - break; - - default: - break; - } -} - - -static inline void -affine_update_priority_heaps(int32 cpu, int32 priority) -{ - int32 core = sCPUToCore[cpu]; - - sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); - - if (sSingleCore) - return; - - int32 maxPriority - = AffineCPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); - int32 corePriority = AffineCorePriorityHeap::GetKey(&sCoreEntries[core]); - - if (corePriority != maxPriority) { - if (maxPriority == B_IDLE_PRIORITY) { - sCorePriorityHeap->ModifyKey(&sCoreEntries[core], B_IDLE_PRIORITY); - ASSERT(sCorePriorityHeap->PeekRoot() == &sCoreEntries[core]); - sCorePriorityHeap->RemoveRoot(); - } else if (corePriority == B_IDLE_PRIORITY) - sCorePriorityHeap->Insert(&sCoreEntries[core], maxPriority); - else - sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); - - int32 package = sCPUToPackage[cpu]; - PackageEntry* packageEntry = &sPackageEntries[package]; - if (maxPriority == B_IDLE_PRIORITY) { - // core goes idle - ASSERT(packageEntry->fIdleCoreCount >= 0); - ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount++; - packageEntry->fIdleCores.Add(&sCoreEntries[core]); - - if (packageEntry->fIdleCoreCount == 1) { - // first core on that package to go idle - - if (packageEntry->fCoreCount > 1) - sPackageUsageHeap->Insert(packageEntry, 1); - else - sIdlePackageList->Add(packageEntry); - } else if (packageEntry->fIdleCoreCount - == packageEntry->fCoreCount) { - // package goes idle - sPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); - sPackageUsageHeap->RemoveMinimum(); - - sIdlePackageList->Add(packageEntry); - } else { - sPackageUsageHeap->ModifyKey(packageEntry, - packageEntry->fIdleCoreCount); - } - } else if (corePriority == B_IDLE_PRIORITY) { - // core wakes up - ASSERT(packageEntry->fIdleCoreCount > 0); - ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount--; - packageEntry->fIdleCores.Remove(&sCoreEntries[core]); - - if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { - // package wakes up - sIdlePackageList->Remove(packageEntry); - - if (packageEntry->fIdleCoreCount > 0) { - sPackageUsageHeap->Insert(packageEntry, - packageEntry->fIdleCoreCount); - } - } else if (packageEntry->fIdleCoreCount == 0) { - // no more idle cores in the package - sPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); - sPackageUsageHeap->RemoveMinimum(); - } else { - sPackageUsageHeap->ModifyKey(packageEntry, - packageEntry->fIdleCoreCount); - } - } - } -} - - -static int32 -affine_choose_core_performance(Thread* thread) -{ - CoreEntry* entry; - - if (sIdlePackageList->Last() != NULL) { - // wake new package - PackageEntry* package = sIdlePackageList->Last(); - entry = package->fIdleCores.Last(); - } else if (sPackageUsageHeap->PeekMaximum() != NULL) { - // wake new core - PackageEntry* package = sPackageUsageHeap->PeekMaximum(); - entry = package->fIdleCores.Last(); - } else { - // no idle cores, use least occupied core - entry = sCorePriorityHeap->PeekRoot(); - - int32 priority = affine_get_effective_priority(thread); - if (AffineCorePriorityHeap::GetKey(entry) >= priority) { - entry = sCoreThreadHeap->PeekMinimum(); - if (entry == NULL) - entry = sCoreCPUBoundThreadHeap->PeekMinimum(); - } - } - - ASSERT(entry != NULL); - return entry->fCoreID; -} - - -static inline bool -affine_is_task_small(Thread* thread) -{ - int32 priority = affine_get_effective_priority(thread); - int32 penalty = thread->scheduler_data->priority_penalty; - return penalty == 0 || priority >= B_DISPLAY_PRIORITY; -} - - -static int32 -affine_choose_core_power_saving(Thread* thread) -{ - CoreEntry* entry; - - int32 priority = affine_get_effective_priority(thread); - - if (sDisableSmallTaskPacking > 0) - sDisableSmallTaskPacking--; - - if (!sDisableSmallTaskPacking && affine_is_task_small(thread) - && sCoreThreadHeap->PeekMaximum() != NULL) { - // try to pack all threads on one core - if (sSmallTaskCore < 0) - sSmallTaskCore = sCoreThreadHeap->PeekMaximum()->fCoreID; - entry = &sCoreEntries[sSmallTaskCore]; - } else if (sCorePriorityHeap->PeekRoot() != NULL - && AffineCorePriorityHeap::GetKey(sCorePriorityHeap->PeekRoot()) - < priority) { - // run immediately on already woken core - entry = sCorePriorityHeap->PeekRoot(); - } else if (sPackageUsageHeap->PeekMinimum() != NULL) { - // wake new core - PackageEntry* package = sPackageUsageHeap->PeekMinimum(); - entry = package->fIdleCores.Last(); - } else if (sIdlePackageList->Last() != NULL) { - // wake new package - PackageEntry* package = sIdlePackageList->Last(); - entry = package->fIdleCores.Last(); - } else { - // no idle cores, use least occupied core - entry = sCoreThreadHeap->PeekMinimum(); - if (entry == NULL) - entry = sCoreCPUBoundThreadHeap->PeekMinimum(); - } - - ASSERT(entry != NULL); - return entry->fCoreID; -} - - -static inline int32 -affine_choose_core(Thread* thread) -{ - ASSERT(!sSingleCore); - return sChooseCore(thread); -} - - -static inline int32 -affine_choose_cpu(int32 core) -{ - CPUEntry* entry = sCPUPriorityHeaps[core].PeekMinimum(); - ASSERT(entry != NULL); - return entry->fCPUNumber; -} - - -static bool -affine_should_rebalance(Thread* thread) -{ - ASSERT(!sSingleCore); - - if (thread_is_idle_thread(thread)) - return false; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; - - // If this is a cpu bound thread and we have significantly more such threads - // than the average get rid of this one. - if (schedulerThreadData->additional_penalty != 0) { - int32 averageCPUBound = sCPUBoundThreads / sRunQueueCount; - return coreEntry->fCPUBoundThreads - averageCPUBound > 1; - } - - // If this thread is not cpu bound but we have at least one consider giving - // this one to someone less busy. - int32 averageThread = sAssignedThreads / sRunQueueCount; - if (coreEntry->fCPUBoundThreads > 0) { - CoreEntry* other = sCoreThreadHeap->PeekMinimum(); - if (other != NULL - && AffineCoreThreadHeap::GetKey(other) <= averageThread) { - return true; - } - } - - int32 threadsAboveAverage = coreEntry->fThreads - averageThread; - - // All cores try to give us small tasks, check whether we have enough. - const int kSmallTaskCountThreshold = 5; - if (sDisableSmallTaskPacking == 0 && sSmallTaskCore == coreEntry->fCoreID) { - if (threadsAboveAverage > kSmallTaskCountThreshold) { - if (!affine_is_task_small(thread)) - return true; - } else if (threadsAboveAverage > 2 * kSmallTaskCountThreshold) { - affine_disable_small_task_packing(); - } - } - - // Try our luck at small task packing. - if (sDisableSmallTaskPacking == 0 && affine_is_task_small(thread)) - return sSmallTaskCore != coreEntry->fCoreID; - - // No cpu bound threads - the situation is quite good. Make sure it - // won't get much worse... - const int32 kBalanceThreshold = 3; - return threadsAboveAverage > kBalanceThreshold; -} - - -static void -affine_assign_active_thread_to_core(Thread* thread) -{ - if (thread_is_idle_thread(thread) || sSingleCore) - return; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - ASSERT(schedulerThreadData->previous_core >= 0); - int32 core = schedulerThreadData->previous_core; - - sCoreEntries[core].fThreads++; - sAssignedThreads++; - - if (schedulerThreadData->additional_penalty != 0) { - sCoreEntries[core].fCPUBoundThreads++; - sCPUBoundThreads++; - } - - affine_update_thread_heaps(core); -} - - -static inline void -affine_thread_goes_away(Thread* thread) -{ - if (thread_is_idle_thread(thread)) - return; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - ASSERT(schedulerThreadData->previous_core >= 0); - int32 core = schedulerThreadData->previous_core; - - schedulerThreadData->went_sleep = system_time(); - schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; - - if (sSingleCore) - return; - - ASSERT(sCoreEntries[core].fThreads > 0); - ASSERT(sCoreEntries[core].fThreads > sCoreEntries[core].fCPUBoundThreads - || (sCoreEntries[core].fThreads == sCoreEntries[core].fCPUBoundThreads - && schedulerThreadData->additional_penalty != 0)); - sCoreEntries[core].fThreads--; - sAssignedThreads--; - - if (schedulerThreadData->additional_penalty != 0) { - ASSERT(sCoreEntries[core].fCPUBoundThreads > 0); - sCoreEntries[core].fCPUBoundThreads--; - sCPUBoundThreads--; - } - - affine_update_thread_heaps(core); -} - - -static void -affine_enqueue(Thread* thread, bool newOne) -{ - ASSERT(thread != NULL); - - thread->state = thread->next_state = B_THREAD_READY; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; - if (newOne && hasSlept > kThreadQuantum) - affine_cancel_penalty(thread); - - int32 threadPriority = affine_get_effective_priority(thread); - - T(EnqueueThread(thread, threadPriority)); - - bool pinned = thread->pinned_to_cpu > 0; - int32 targetCPU = -1; - int32 targetCore; - if (pinned) { - targetCPU = thread->previous_cpu->cpu_num; - targetCore = sCPUToCore[targetCPU]; - ASSERT(targetCore == schedulerThreadData->previous_core); - - if (newOne) - affine_assign_active_thread_to_core(thread); - } else if (sSingleCore) { - targetCore = 0; - targetCPU = affine_choose_cpu(targetCore); - - schedulerThreadData->previous_core = targetCore; - } else if (schedulerThreadData->previous_core < 0 - || (newOne && affine_has_cache_expired(thread)) - || affine_should_rebalance(thread)) { - - if (thread_is_idle_thread(thread)) { - targetCPU = thread->previous_cpu->cpu_num; - targetCore = sCPUToCore[targetCPU]; - } else { - if (!newOne) - affine_thread_goes_away(thread); - - targetCore = affine_choose_core(thread); - targetCPU = affine_choose_cpu(targetCore); - } - - schedulerThreadData->previous_core = targetCore; - affine_assign_active_thread_to_core(thread); - } else { - targetCore = schedulerThreadData->previous_core; - targetCPU = affine_choose_cpu(targetCore); - if (newOne) - affine_assign_active_thread_to_core(thread); - } - - ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); - ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); - - TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, - threadPriority, targetCore); - if (pinned) - sPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); - else - sRunQueues[targetCore].PushBack(thread, threadPriority); - - schedulerThreadData->cpu_bound = true; - schedulerThreadData->time_left = 0; - schedulerThreadData->stolen_time = 0; - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, - thread); - - Thread* targetThread = gCPU[targetCPU].running_thread; - int32 targetPriority = affine_get_effective_priority(targetThread); - - TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, - targetPriority); - - if (threadPriority > targetPriority) { - targetThread->scheduler_data->lost_cpu = true; - - // It is possible that another CPU schedules the thread before the - // target CPU. However, since the target CPU is sent an ICI it will - // reschedule anyway and update its heap key to the correct value. - affine_update_priority_heaps(targetCPU, threadPriority); - - if (targetCPU == smp_get_current_cpu()) - gCPU[targetCPU].invoke_scheduler = true; - else { - smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, - SMP_MSG_FLAG_ASYNC); - } - } -} - - -/*! Enqueues the thread into the run queue. - Note: thread lock must be held when entering this function -*/ -static void -affine_enqueue_in_run_queue(Thread *thread) -{ - TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, - thread->priority); - affine_enqueue(thread, true); -} - - -static inline void -affine_put_back(Thread* thread) -{ - bool pinned = sPinnedRunQueues != NULL && thread->pinned_to_cpu > 0; - - if (pinned) { - int32 pinnedCPU = thread->previous_cpu->cpu_num; - sPinnedRunQueues[pinnedCPU].PushFront(thread, - affine_get_effective_priority(thread)); - } else { - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); - sRunQueues[previousCore].PushFront(thread, - affine_get_effective_priority(thread)); - } -} - - -/*! Sets the priority of a thread. - Note: thread lock must be held when entering this function -*/ -static void -affine_set_thread_priority(Thread *thread, int32 priority) -{ - if (priority == thread->priority) - return; - - TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", - thread->id, priority, thread->priority, - affine_get_effective_priority(thread)); - - if (thread->state == B_THREAD_RUNNING) - affine_thread_goes_away(thread); - - if (thread->state != B_THREAD_READY) { - affine_cancel_penalty(thread); - thread->priority = priority; - - if (thread->state == B_THREAD_RUNNING) { - affine_assign_active_thread_to_core(thread); - affine_update_priority_heaps(thread->cpu->cpu_num, priority); - } - return; - } - - // The thread is in the run queue. We need to remove it and re-insert it at - // a new position. - - T(RemoveThread(thread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, - thread); - - // remove thread from run queue - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); - sRunQueues[previousCore].Remove(thread); - affine_thread_goes_away(thread); - - // set priority and re-insert - affine_cancel_penalty(thread); - thread->priority = priority; - - affine_enqueue_in_run_queue(thread); -} - - -static bigtime_t -affine_estimate_max_scheduling_latency(Thread* thread) -{ - // TODO: This is probably meant to be called periodically to return the - // current estimate depending on the system usage; we return fixed estimates - // per thread priority, though. - - if (thread->priority >= B_REAL_TIME_DISPLAY_PRIORITY) - return kMinThreadQuantum / 4; - if (thread->priority >= B_DISPLAY_PRIORITY) - return kMinThreadQuantum; - if (thread->priority < B_NORMAL_PRIORITY) - return 2 * kMaxThreadQuantum; - - return 2 * kMinThreadQuantum; -} - - -static int32 -reschedule_event(timer *unused) -{ - // This function is called as a result of the timer event set by the - // scheduler. Make sure the reschedule() is invoked. - Thread* thread= thread_get_current_thread(); - - thread->scheduler_data->lost_cpu = true; - thread->cpu->invoke_scheduler = true; - thread->cpu->preempted = 1; - return B_HANDLED_INTERRUPT; -} - - -static inline bool -affine_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (hasYielded) { - schedulerThreadData->time_left = 0; - return true; - } - - bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; - schedulerThreadData->time_left -= time_used; - schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); - - // too little time left, it's better make the next quantum a bit longer - if (wasPreempted || schedulerThreadData->time_left <= kThreadQuantum / 50) { - schedulerThreadData->stolen_time += schedulerThreadData->time_left; - schedulerThreadData->time_left = 0; - } - - return schedulerThreadData->time_left == 0; -} - - -static inline bigtime_t -affine_quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, - int32 maxPriority, int32 minPriority, int32 priority) -{ - ASSERT(priority <= maxPriority); - ASSERT(priority >= minPriority); - - bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); - result /= maxPriority - minPriority; - return maxQuantum - result; -} - - -static inline bigtime_t -affine_get_base_quantum(Thread* thread) -{ - int32 priority = affine_get_effective_priority(thread); - - if (priority >= B_URGENT_DISPLAY_PRIORITY) - return kThreadQuantum; - if (priority > B_NORMAL_PRIORITY) { - return affine_quantum_linear_interpolation(kThreadQuantum * 4, - kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, - priority); - } - return affine_quantum_linear_interpolation(kThreadQuantum * 64, - kThreadQuantum * 4, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); -} - - -static inline bigtime_t -affine_compute_quantum(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - bigtime_t quantum; - if (schedulerThreadData->time_left != 0) - quantum = schedulerThreadData->time_left; - else - quantum = affine_get_base_quantum(thread); - - quantum += schedulerThreadData->stolen_time; - schedulerThreadData->stolen_time = 0; - - schedulerThreadData->time_left = quantum; - schedulerThreadData->quantum_start = system_time(); - - return quantum; -} - - -static inline Thread* -affine_dequeue_thread(int32 thisCPU) -{ - int32 thisCore = sCPUToCore[thisCPU]; - Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); - - Thread* pinnedThread = NULL; - if (sPinnedRunQueues != NULL) - pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); - - if (sharedThread == NULL && pinnedThread == NULL) - return NULL; - - int32 pinnedPriority = -1; - if (pinnedThread != NULL) - pinnedPriority = affine_get_effective_priority(pinnedThread); - - int32 sharedPriority = -1; - if (sharedThread != NULL) - sharedPriority = affine_get_effective_priority(sharedThread); - - if (sharedPriority > pinnedPriority) { - sRunQueues[thisCore].Remove(sharedThread); - return sharedThread; - } - - sPinnedRunQueues[thisCPU].Remove(pinnedThread); - return pinnedThread; -} - - -static inline void -affine_track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) -{ - if (!thread_is_idle_thread(oldThread)) { - bigtime_t active - = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time); - - atomic_add64(&oldThread->cpu->active_time, active); - sCoreEntries[thisCore].fActiveTime += active; - } - - if (!thread_is_idle_thread(nextThread)) { - oldThread->cpu->last_kernel_time = nextThread->kernel_time; - oldThread->cpu->last_user_time = nextThread->user_time; - } -} - - -/*! Runs the scheduler. - Note: expects thread spinlock to be held -*/ -static void -affine_reschedule(void) -{ - Thread* oldThread = thread_get_current_thread(); - - int32 thisCPU = smp_get_current_cpu(); - int32 thisCore = sCPUToCore[thisCPU]; - - TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, - oldThread->id); - - oldThread->state = oldThread->next_state; - scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; - - // update CPU heap so that old thread would have CPU properly chosen - Thread* nextThread = sRunQueues[thisCore].PeekMaximum(); - if (nextThread != NULL) { - affine_update_priority_heaps(thisCPU, - affine_get_effective_priority(nextThread)); - } - - switch (oldThread->next_state) { - case B_THREAD_RUNNING: - case B_THREAD_READY: - if (!schedulerOldThreadData->lost_cpu) - schedulerOldThreadData->cpu_bound = false; - - if (affine_quantum_ended(oldThread, oldThread->cpu->preempted, - oldThread->has_yielded)) { - if (schedulerOldThreadData->cpu_bound) - affine_increase_penalty(oldThread); - - TRACE("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, affine_get_effective_priority(oldThread)); - affine_enqueue(oldThread, false); - } else { - TRACE("putting thread %ld back in run queue priority = %ld\n", - oldThread->id, affine_get_effective_priority(oldThread)); - affine_put_back(oldThread); - } - - break; - case B_THREAD_SUSPENDED: - affine_thread_goes_away(oldThread); - TRACE("reschedule(): suspending thread %ld\n", oldThread->id); - break; - case THREAD_STATE_FREE_ON_RESCHED: - affine_thread_goes_away(oldThread); - break; - default: - affine_thread_goes_away(oldThread); - TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", - oldThread->id, oldThread->next_state); - break; - } - - oldThread->has_yielded = false; - schedulerOldThreadData->lost_cpu = false; - - // select thread with the biggest priority - if (oldThread->cpu->disabled) { - ASSERT(sPinnedRunQueues != NULL); - nextThread = sPinnedRunQueues[thisCPU].PeekMaximum(); - if (nextThread != NULL) - sPinnedRunQueues[thisCPU].Remove(nextThread); - else { - nextThread = sRunQueues[thisCore].GetHead(B_IDLE_PRIORITY); - if (nextThread != NULL) - sRunQueues[thisCore].Remove(nextThread); - } - } else - nextThread = affine_dequeue_thread(thisCPU); - if (!nextThread) - panic("reschedule(): run queues are empty!\n"); - - TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, - nextThread->id); - - T(ScheduleThread(nextThread, oldThread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, - oldThread, nextThread); - - // update CPU heap - affine_update_priority_heaps(thisCPU, - affine_get_effective_priority(nextThread)); - - nextThread->state = B_THREAD_RUNNING; - nextThread->next_state = B_THREAD_READY; - ASSERT(nextThread->scheduler_data->previous_core == thisCore); - //nextThread->scheduler_data->previous_core = thisCore; - - // track kernel time (user time is tracked in thread_at_kernel_entry()) - scheduler_update_thread_times(oldThread, nextThread); - - // track CPU activity - affine_track_cpu_activity(oldThread, nextThread, thisCore); - - if (nextThread != oldThread || oldThread->cpu->preempted) { - timer* quantumTimer = &oldThread->cpu->quantum_timer; - if (!oldThread->cpu->preempted) - cancel_timer(quantumTimer); - - oldThread->cpu->preempted = 0; - if (!thread_is_idle_thread(nextThread)) { - bigtime_t quantum = affine_compute_quantum(oldThread); - add_timer(quantumTimer, &reschedule_event, quantum, - B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); - } - - if (nextThread != oldThread) - scheduler_switch_thread(oldThread, nextThread); - } -} - - -static status_t -affine_on_thread_create(Thread* thread, bool idleThread) -{ - thread->scheduler_data = new (std::nothrow)scheduler_thread_data; - if (thread->scheduler_data == NULL) - return B_NO_MEMORY; - return B_OK; -} - - -static void -affine_on_thread_init(Thread* thread) -{ - thread->scheduler_data->Init(); -} - - -static void -affine_on_thread_destroy(Thread* thread) -{ - delete thread->scheduler_data; -} - - -/*! This starts the scheduler. Must be run in the context of the initial idle - thread. Interrupts must be disabled and will be disabled when returning. -*/ -static void -affine_start(void) -{ - SpinLocker schedulerLocker(gSchedulerLock); - - affine_reschedule(); -} - - -static status_t -affine_set_operation_mode(scheduler_mode mode) -{ - if (mode != SCHEDULER_MODE_PERFORMANCE - && mode != SCHEDULER_MODE_POWER_SAVING) { - return B_BAD_VALUE; - } - -#ifdef TRACE_SCHEDULER - const char* modeNames = { "performance", "power saving" }; -#endif - TRACE("switching scheduler to %s mode\n", modeNames[mode]); - - sSchedulerMode = mode; - switch (mode) { - case SCHEDULER_MODE_PERFORMANCE: - sDisableSmallTaskPacking = -1; - sSmallTaskCore = -1; - sChooseCore = affine_choose_core_performance; - break; - - case SCHEDULER_MODE_POWER_SAVING: - sDisableSmallTaskPacking = 0; - sSmallTaskCore = -1; - sChooseCore = affine_choose_core_power_saving; - break; - - default: - break; - } - - return B_OK; -} - - -static scheduler_ops kAffineOps = { - affine_enqueue_in_run_queue, - affine_reschedule, - affine_set_thread_priority, - affine_estimate_max_scheduling_latency, - affine_on_thread_create, - affine_on_thread_init, - affine_on_thread_destroy, - affine_start, - affine_set_operation_mode, - affine_dump_thread_data -}; - - -// #pragma mark - - - -static void -traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) -{ - switch (node->level) { - case CPU_TOPOLOGY_SMT: - sCPUToCore[node->id] = coreID; - sCPUToPackage[node->id] = packageID; - return; - - case CPU_TOPOLOGY_CORE: - coreID = node->id; - break; - - case CPU_TOPOLOGY_PACKAGE: - packageID = node->id; - break; - - default: - break; - } - - for (int32 i = 0; i < node->children_count; i++) - traverse_topology_tree(node->children[i], packageID, coreID); -} - - -static status_t -build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) -{ - cpuCount = smp_get_num_cpus(); - - sCPUToCore = new(std::nothrow) int32[cpuCount]; - if (sCPUToCore == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuToCoreDeleter(sCPUToCore); - - sCPUToPackage = new(std::nothrow) int32[cpuCount]; - if (sCPUToPackage == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuToPackageDeleter(sCPUToPackage); - - coreCount = 0; - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) - coreCount++; - } - - packageCount = 0; - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0 - && gCPU[i].topology_id[CPU_TOPOLOGY_CORE] == 0) { - packageCount++; - } - } - - cpu_topology_node* root = get_cpu_topology(); - traverse_topology_tree(root, 0, 0); - - cpuToCoreDeleter.Detach(); - cpuToPackageDeleter.Detach(); - return B_OK; -} - - -status_t -scheduler_affine_init() -{ - // create logical processor to core and package mappings - int32 cpuCount, coreCount, packageCount; - status_t result = build_topology_mappings(cpuCount, coreCount, - packageCount); - if (result != B_OK) - return result; - sRunQueueCount = coreCount; - sSingleCore = coreCount == 1; - - // create package heap and idle package stack - sPackageEntries = new(std::nothrow) PackageEntry[packageCount]; - if (sPackageEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter packageEntriesDeleter(sPackageEntries); - - sPackageUsageHeap = new(std::nothrow) AffinePackageHeap(packageCount); - if (sPackageUsageHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter packageHeapDeleter(sPackageUsageHeap); - - sIdlePackageList = new(std::nothrow) AffineIdlePackageList; - if (sIdlePackageList == NULL) - return B_NO_MEMORY; - ObjectDeleter packageListDeleter(sIdlePackageList); - - for (int32 i = 0; i < packageCount; i++) { - sPackageEntries[i].fPackageID = i; - sPackageEntries[i].fIdleCoreCount = coreCount / packageCount; - sPackageEntries[i].fCoreCount = coreCount / packageCount; - sIdlePackageList->Insert(&sPackageEntries[i]); - } - - // create logical processor and core heaps - sCPUEntries = new CPUEntry[cpuCount]; - if (sCPUEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuEntriesDeleter(sCPUEntries); - - sCoreEntries = new CoreEntry[coreCount]; - if (sCoreEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter coreEntriesDeleter(sCoreEntries); - - sCorePriorityHeap = new AffineCorePriorityHeap(coreCount); - if (sCorePriorityHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter corePriorityHeapDeleter( - sCorePriorityHeap); - - sCoreThreadHeap = new AffineCoreThreadHeap; - if (sCoreThreadHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter coreThreadHeapDeleter(sCoreThreadHeap); - - sCoreCPUBoundThreadHeap = new AffineCoreThreadHeap(coreCount); - if (sCoreCPUBoundThreadHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter coreCPUThreadHeapDeleter( - sCoreCPUBoundThreadHeap); - - for (int32 i = 0; i < coreCount; i++) { - sCoreEntries[i].fCoreID = i; - sCoreEntries[i].fActiveTime = 0; - sCoreEntries[i].fThreads = 0; - sCoreEntries[i].fCPUBoundThreads = 0; - - status_t result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); - if (result != B_OK) - return result; - - result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; - sCorePriorityHeap->RemoveRoot(); - } - - sCPUPriorityHeaps = new AffineCPUHeap[coreCount]; - if (sCPUPriorityHeaps == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); - - for (int32 i = 0; i < cpuCount; i++) { - sCPUEntries[i].fCPUNumber = i; - int32 core = sCPUToCore[i]; - - int32 package = sCPUToPackage[i]; - if (sCPUPriorityHeaps[core].PeekMaximum() == NULL) - sPackageEntries[package].fIdleCores.Insert(&sCoreEntries[core]); - - status_t result - = sCPUPriorityHeaps[core].Insert(&sCPUEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; - } - - // create per-logical processor run queues for pinned threads - TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-cpu queue%s\n", - cpuCount, cpuCount != 1 ? "s" : ""); - - sPinnedRunQueues = new(std::nothrow) AffineRunQueue[cpuCount]; - if (sPinnedRunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter pinnedRunQueuesDeleter(sPinnedRunQueues); - for (int i = 0; i < cpuCount; i++) { - status_t result = sPinnedRunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - - // create per-core run queues - TRACE("scheduler_affine_init(): creating %" B_PRId32 " per-core queue%s\n", - coreCount, coreCount != 1 ? "s" : ""); - - sRunQueues = new(std::nothrow) AffineRunQueue[coreCount]; - if (sRunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter runQueuesDeleter(sRunQueues); - for (int i = 0; i < coreCount; i++) { - status_t result = sRunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - -#if 1 - affine_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); -#else - affine_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); -#endif - gScheduler = &kAffineOps; - - add_debugger_command_etc("run_queue", &dump_run_queue, - "List threads in run queue", "\nLists threads in run queue", 0); - add_debugger_command_etc("cpu_heap", &dump_cpu_heap, - "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", - 0); - if (!sSingleCore) { - add_debugger_command_etc("idle_cores", &dump_idle_cores, - "List idle cores", "\nList idle cores", 0); - } - - runQueuesDeleter.Detach(); - pinnedRunQueuesDeleter.Detach(); - coreCPUThreadHeapDeleter.Detach(); - coreThreadHeapDeleter.Detach(); - corePriorityHeapDeleter.Detach(); - cpuPriorityHeapDeleter.Detach(); - coreEntriesDeleter.Detach(); - cpuEntriesDeleter.Detach(); - packageEntriesDeleter.Detach(); - packageHeapDeleter.Detach(); - packageListDeleter.Detach(); - return B_OK; -} diff --git a/src/system/kernel/scheduler/scheduler_affine.h b/src/system/kernel/scheduler/scheduler_affine.h deleted file mode 100644 index 3c52cd06fe..0000000000 --- a/src/system/kernel/scheduler/scheduler_affine.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Copyright 2009, Rene Gollent, rene@gollent.com. - * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. - * Distributed under the terms of the MIT License. - */ -#ifndef KERNEL_SCHEDULER_AFFINE_H -#define KERNEL_SCHEDULER_AFFINE_H - - -status_t scheduler_affine_init(); - - -#endif // KERNEL_SCHEDULER_AFFINE_H diff --git a/src/system/kernel/scheduler/scheduler_simple.cpp b/src/system/kernel/scheduler/scheduler_simple.cpp deleted file mode 100644 index c2b2e3aedb..0000000000 --- a/src/system/kernel/scheduler/scheduler_simple.cpp +++ /dev/null @@ -1,776 +0,0 @@ -/* - * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org - * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. - * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. - * Copyright 2002, Angelo Mottola, a.mottola@libero.it. - * Distributed under the terms of the MIT License. - * - * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. - * Distributed under the terms of the NewOS License. - */ - - -/*! The thread scheduler */ - - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "RunQueue.h" -#include "scheduler_common.h" -#include "scheduler_tracing.h" - - -//#define TRACE_SCHEDULER -#ifdef TRACE_SCHEDULER -# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) -#else -# define TRACE(...) do { } while (false) -#endif - - -const bigtime_t kThreadQuantum = 1000; - - -struct CPUHeapEntry : public HeapLinkImpl { - int32 fCPUNumber; -}; - -static CPUHeapEntry* sCPUEntries; -typedef Heap SimpleCPUHeap; -static SimpleCPUHeap* sCPUHeap; - -// The run queue. Holds the threads ready to run ordered by priority. -typedef RunQueue SimpleRunQueue; -static SimpleRunQueue* sRunQueue; -static SimpleRunQueue* sCPURunQueues; - - -struct scheduler_thread_data { - scheduler_thread_data() { Init(); } - void Init(); - - int32 priority_penalty; - int32 additional_penalty; - - bool lost_cpu; - bool cpu_bound; - - bigtime_t time_left; - bigtime_t stolen_time; - bigtime_t quantum_start; - - bigtime_t went_sleep; -}; - - -void -scheduler_thread_data::Init() -{ - priority_penalty = 0; - additional_penalty = 0; - - time_left = 0; - stolen_time = 0; - - went_sleep = 0; - - lost_cpu = false; - cpu_bound = true; -} - - -static inline int -simple_get_minimal_priority(Thread* thread) -{ - return min_c(thread->priority, 25) / 5; -} - - -static inline int32 -simple_get_thread_penalty(Thread* thread) -{ - int32 penalty = thread->scheduler_data->priority_penalty; - - const int kMinimalPriority = simple_get_minimal_priority(thread); - if (kMinimalPriority > 0) { - penalty - += thread->scheduler_data->additional_penalty % kMinimalPriority; - } - - return penalty; - -} - - -static inline int32 -simple_get_effective_priority(Thread* thread) -{ - if (thread->priority == B_IDLE_PRIORITY) - return thread->priority; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return thread->priority; - - int32 effectivePriority = thread->priority; - effectivePriority -= simple_get_thread_penalty(thread); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; -} - - -static void -dump_queue(SimpleRunQueue::ConstIterator& iterator) -{ - if (!iterator.HasNext()) - kprintf("Run queue is empty.\n"); - else { - kprintf("thread id priority penalty name\n"); - while (iterator.HasNext()) { - Thread* thread = iterator.Next(); - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", - thread, thread->id, thread->priority, - simple_get_thread_penalty(thread), thread->name); - } - } -} - - -static int -dump_run_queue(int argc, char** argv) -{ - SimpleRunQueue::ConstIterator iterator = sRunQueue->GetConstIterator(); - kprintf("Shared run queue:\n"); - dump_queue(iterator); - - int32 cpuCount = smp_get_num_cpus(); - if (cpuCount < 2) - return 0; - - for (int32 i = 0; i < cpuCount; i++) { - iterator = sCPURunQueues[i].GetConstIterator(); - - if (iterator.HasNext()) { - kprintf("\nCPU %" B_PRId32 " run queue:\n", i); - dump_queue(iterator); - } - } - - return 0; -} - - -static int -dump_cpu_heap(int argc, char** argv) -{ - kprintf("cpu priority actual priority\n"); - CPUHeapEntry* entry = sCPUHeap->PeekRoot(); - while (entry) { - int32 cpu = entry->fCPUNumber; - kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, - sCPUHeap->GetKey(entry), - simple_get_effective_priority(gCPU[cpu].running_thread)); - - sCPUHeap->RemoveRoot(); - entry = sCPUHeap->PeekRoot(); - } - - int32 cpuCount = smp_get_num_cpus(); - for (int i = 0; i < cpuCount; i++) { - sCPUHeap->Insert(&sCPUEntries[i], - simple_get_effective_priority(gCPU[i].running_thread)); - } - - return 0; -} - - -static void -simple_dump_thread_data(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", - schedulerThreadData->priority_penalty); - - int32 additionalPenalty = 0; - const int kMinimalPriority = simple_get_minimal_priority(thread); - if (kMinimalPriority > 0) { - additionalPenalty - = schedulerThreadData->additional_penalty % kMinimalPriority; - } - kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", - additionalPenalty, schedulerThreadData->additional_penalty); - kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", - schedulerThreadData->stolen_time); -} - - -static inline void -simple_increase_penalty(Thread* thread) -{ - if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) - return; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return; - - TRACE("increasing thread %ld penalty\n", thread->id); - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - int32 oldPenalty = schedulerThreadData->priority_penalty++; - - ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - const int kMinimalPriority = simple_get_minimal_priority(thread); - if (thread->priority - oldPenalty <= kMinimalPriority) { - schedulerThreadData->priority_penalty = oldPenalty; - schedulerThreadData->additional_penalty++; - } -} - - -static inline void -simple_cancel_penalty(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (schedulerThreadData->priority_penalty != 0) - TRACE("cancelling thread %ld penalty\n", thread->id); - schedulerThreadData->priority_penalty = 0; - schedulerThreadData->additional_penalty = 0; -} - - -static void -simple_enqueue(Thread* thread, bool newOne) -{ - thread->state = thread->next_state = B_THREAD_READY; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; - if (newOne && hasSlept > kThreadQuantum) - simple_cancel_penalty(thread); - - int32 threadPriority = simple_get_effective_priority(thread); - - T(EnqueueThread(thread, threadPriority)); - - bool pinned = sCPURunQueues != NULL && thread->pinned_to_cpu > 0; - int32 pinnedCPU = -1; - if (pinned) { - pinnedCPU = thread->previous_cpu->cpu_num; - sCPURunQueues[pinnedCPU].PushBack(thread, threadPriority); - } else - sRunQueue->PushBack(thread, threadPriority); - - schedulerThreadData->cpu_bound = true; - schedulerThreadData->time_left = 0; - schedulerThreadData->stolen_time = 0; - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, - thread); - - int32 thisCPU = smp_get_current_cpu(); - int32 targetCPU = pinnedCPU; - - if (!pinned) { - CPUHeapEntry* cpuEntry = sCPUHeap->PeekRoot(); - ASSERT(cpuEntry != NULL); - - targetCPU = cpuEntry->fCPUNumber; - } - - ASSERT(targetCPU >= 0); - - Thread* targetThread = gCPU[targetCPU].running_thread; - int32 targetPriority = simple_get_effective_priority(targetThread); - - ASSERT((targetCPU != thisCPU && targetThread != thread) - || targetCPU == thisCPU); - - if (!pinned) { - int32 currentThreadPriority - = simple_get_effective_priority(thread_get_current_thread()); - if (targetPriority == currentThreadPriority) { - targetCPU = thisCPU; - targetPriority = currentThreadPriority; - } - } - - TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, - targetPriority); - - if (threadPriority > targetPriority) { - targetThread->scheduler_data->lost_cpu = true; - - // It is possible that another CPU schedules the thread before the - // target CPU. However, since the target CPU is sent an ICI it will - // reschedule anyway and update its heap key to the correct value. - sCPUHeap->ModifyKey(&sCPUEntries[targetCPU], threadPriority); - - if (targetCPU == smp_get_current_cpu()) - gCPU[targetCPU].invoke_scheduler = true; - else { - smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, - SMP_MSG_FLAG_ASYNC); - } - } -} - - -/*! Enqueues the thread into the run queue. - Note: thread lock must be held when entering this function -*/ -static void -simple_enqueue_in_run_queue(Thread* thread) -{ - TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, - thread->priority); - simple_enqueue(thread, true); -} - - -static inline void -simple_put_back(Thread* thread) -{ - bool pinned = sCPURunQueues != NULL && thread->pinned_to_cpu > 0; - - if (!pinned) - sRunQueue->PushFront(thread, simple_get_effective_priority(thread)); - else { - int32 pinnedCPU = thread->previous_cpu->cpu_num; - sCPURunQueues[pinnedCPU].PushFront(thread, - simple_get_effective_priority(thread)); - } -} - - -/*! Sets the priority of a thread. - Note: thread lock must be held when entering this function -*/ -static void -simple_set_thread_priority(Thread* thread, int32 priority) -{ - if (priority == thread->priority) - return; - - TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", - thread->id, priority, thread->priority, - simple_get_effective_priority(thread)); - - if (thread->state == B_THREAD_RUNNING) - sCPUHeap->ModifyKey(&sCPUEntries[thread->cpu->cpu_num], priority); - - if (thread->state != B_THREAD_READY) { - simple_cancel_penalty(thread); - thread->priority = priority; - return; - } - - // The thread is in the run queue. We need to remove it and re-insert it at - // a new position. - - T(RemoveThread(thread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, - thread); - - // remove thread from run queue - sRunQueue->Remove(thread); - - // set priority and re-insert - simple_cancel_penalty(thread); - thread->priority = priority; - - simple_enqueue_in_run_queue(thread); -} - - -static bigtime_t -simple_estimate_max_scheduling_latency(Thread* thread) -{ - // TODO: This is probably meant to be called periodically to return the - // current estimate depending on the system usage; we return fixed estimates - // per thread priority, though. - - if (thread->priority >= B_REAL_TIME_DISPLAY_PRIORITY) - return kThreadQuantum / 4; - if (thread->priority >= B_DISPLAY_PRIORITY) - return kThreadQuantum; - - return 2 * kThreadQuantum; -} - - -static int32 -reschedule_event(timer* /* unused */) -{ - // This function is called as a result of the timer event set by the - // scheduler. Make sure the reschedule() is invoked. - Thread* thread= thread_get_current_thread(); - - thread->scheduler_data->lost_cpu = true; - thread->cpu->invoke_scheduler = true; - thread->cpu->preempted = 1; - return B_HANDLED_INTERRUPT; -} - - -static inline bool -simple_quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (hasYielded) { - schedulerThreadData->time_left = 0; - return true; - } - - bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; - schedulerThreadData->time_left -= time_used; - schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); - - // too little time left, it's better make the next quantum a bit longer - if (wasPreempted || schedulerThreadData->time_left <= kThreadQuantum / 50) { - schedulerThreadData->stolen_time += schedulerThreadData->time_left; - schedulerThreadData->time_left = 0; - } - - return schedulerThreadData->time_left == 0; -} - - -static inline bigtime_t -simple_quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, - int32 maxPriority, int32 minPriority, int32 priority) -{ - ASSERT(priority <= maxPriority); - ASSERT(priority >= minPriority); - - bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); - result /= maxPriority - minPriority; - return maxQuantum - result; -} - - -static inline bigtime_t -simple_get_base_quantum(Thread* thread) -{ - int32 priority = simple_get_effective_priority(thread); - - if (priority >= B_URGENT_DISPLAY_PRIORITY) - return kThreadQuantum; - if (priority > B_NORMAL_PRIORITY) { - return simple_quantum_linear_interpolation(kThreadQuantum * 4, - kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, - priority); - } - return simple_quantum_linear_interpolation(kThreadQuantum * 64, - kThreadQuantum * 4, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); -} - - -static inline bigtime_t -simple_compute_quantum(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - bigtime_t quantum; - if (schedulerThreadData->time_left != 0) - quantum = schedulerThreadData->time_left; - else - quantum = simple_get_base_quantum(thread); - - quantum += schedulerThreadData->stolen_time; - schedulerThreadData->stolen_time = 0; - - schedulerThreadData->time_left = quantum; - schedulerThreadData->quantum_start = system_time(); - - return quantum; -} - - -static inline Thread* -simple_dequeue_thread(int32 thisCPU) -{ - Thread* sharedThread = sRunQueue->PeekMaximum(); - - Thread* pinnedThread = NULL; - if (sCPURunQueues != NULL) - pinnedThread = sCPURunQueues[thisCPU].PeekMaximum(); - - if (sharedThread == NULL && pinnedThread == NULL) - return NULL; - - int32 pinnedPriority = -1; - if (pinnedThread != NULL) - pinnedPriority = simple_get_effective_priority(pinnedThread); - - int32 sharedPriority = -1; - if (sharedThread != NULL) - sharedPriority = simple_get_effective_priority(sharedThread); - - if (sharedPriority > pinnedPriority) { - sRunQueue->Remove(sharedThread); - return sharedThread; - } - - sCPURunQueues[thisCPU].Remove(pinnedThread); - return pinnedThread; -} - - -/*! Runs the scheduler. - Note: expects thread spinlock to be held -*/ -static void -simple_reschedule(void) -{ - Thread* oldThread = thread_get_current_thread(); - - int32 thisCPU = smp_get_current_cpu(); - - TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, - oldThread->id); - - oldThread->state = oldThread->next_state; - scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; - - // update CPU heap so that old thread would have CPU properly chosen - Thread* nextThread = sRunQueue->PeekMaximum(); - if (nextThread != NULL) { - sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], - simple_get_effective_priority(nextThread)); - } - - switch (oldThread->next_state) { - case B_THREAD_RUNNING: - case B_THREAD_READY: - if (!schedulerOldThreadData->lost_cpu) - schedulerOldThreadData->cpu_bound = false; - - if (simple_quantum_ended(oldThread, oldThread->cpu->preempted, - oldThread->has_yielded)) { - if (schedulerOldThreadData->cpu_bound) - simple_increase_penalty(oldThread); - - TRACE("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, simple_get_effective_priority(oldThread)); - simple_enqueue(oldThread, false); - } else { - TRACE("putting thread %ld back in run queue priority = %ld\n", - oldThread->id, simple_get_effective_priority(oldThread)); - simple_put_back(oldThread); - } - - break; - case B_THREAD_SUSPENDED: - schedulerOldThreadData->went_sleep = system_time(); - TRACE("reschedule(): suspending thread %ld\n", oldThread->id); - break; - case THREAD_STATE_FREE_ON_RESCHED: - break; - default: - schedulerOldThreadData->went_sleep = system_time(); - TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", - oldThread->id, oldThread->next_state); - break; - } - - oldThread->has_yielded = false; - schedulerOldThreadData->lost_cpu = false; - - // select thread with the biggest priority - if (oldThread->cpu->disabled) { - ASSERT(sCPURunQueues != NULL); - nextThread = sCPURunQueues[thisCPU].PeekMaximum(); - if (nextThread != NULL) - sCPURunQueues[thisCPU].Remove(nextThread); - else { - nextThread = sRunQueue->GetHead(B_IDLE_PRIORITY); - if (nextThread != NULL) - sRunQueue->Remove(nextThread); - } - } else - nextThread = simple_dequeue_thread(thisCPU); - if (!nextThread) - panic("reschedule(): run queues are empty!\n"); - - TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, - nextThread->id); - - T(ScheduleThread(nextThread, oldThread)); - - // update CPU heap - sCPUHeap->ModifyKey(&sCPUEntries[thisCPU], - simple_get_effective_priority(nextThread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, - oldThread, nextThread); - - nextThread->state = B_THREAD_RUNNING; - nextThread->next_state = B_THREAD_READY; - - // track kernel time (user time is tracked in thread_at_kernel_entry()) - scheduler_update_thread_times(oldThread, nextThread); - - // track CPU activity - if (!thread_is_idle_thread(oldThread)) { - atomic_add64(&oldThread->cpu->active_time, - (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time)); - } - - if (!thread_is_idle_thread(nextThread)) { - oldThread->cpu->last_kernel_time = nextThread->kernel_time; - oldThread->cpu->last_user_time = nextThread->user_time; - } - - if (nextThread != oldThread || oldThread->cpu->preempted) { - timer* quantumTimer = &oldThread->cpu->quantum_timer; - if (!oldThread->cpu->preempted) - cancel_timer(quantumTimer); - - oldThread->cpu->preempted = 0; - if (!thread_is_idle_thread(nextThread)) { - bigtime_t quantum = simple_compute_quantum(oldThread); - add_timer(quantumTimer, &reschedule_event, quantum, - B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); - } - - if (nextThread != oldThread) - scheduler_switch_thread(oldThread, nextThread); - } -} - - -static status_t -simple_on_thread_create(Thread* thread, bool idleThread) -{ - thread->scheduler_data = new (std::nothrow)scheduler_thread_data; - if (thread->scheduler_data == NULL) - return B_NO_MEMORY; - return B_OK; -} - - -static void -simple_on_thread_init(Thread* thread) -{ - thread->scheduler_data->Init(); -} - - -static void -simple_on_thread_destroy(Thread* thread) -{ - delete thread->scheduler_data; -} - - -/*! This starts the scheduler. Must be run in the context of the initial idle - thread. Interrupts must be disabled and will be disabled when returning. -*/ -static void -simple_start(void) -{ - SpinLocker schedulerLocker(gSchedulerLock); - - simple_reschedule(); -} - - -static scheduler_ops kSimpleOps = { - simple_enqueue_in_run_queue, - simple_reschedule, - simple_set_thread_priority, - simple_estimate_max_scheduling_latency, - simple_on_thread_create, - simple_on_thread_init, - simple_on_thread_destroy, - simple_start, - NULL, - simple_dump_thread_data -}; - - -// #pragma mark - - - -status_t -scheduler_simple_init() -{ - int32 cpuCount = smp_get_num_cpus(); - - sCPUHeap = new SimpleCPUHeap; - if (sCPUHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter cpuHeapDeleter(sCPUHeap); - - sCPUEntries = new CPUHeapEntry[cpuCount]; - if (sCPUEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuEntriesDeleter(sCPUEntries); - - for (int i = 0; i < cpuCount; i++) { - sCPUEntries[i].fCPUNumber = i; - status_t result = sCPUHeap->Insert(&sCPUEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; - } - - sRunQueue = new(std::nothrow) SimpleRunQueue; - if (sRunQueue == NULL) - return B_NO_MEMORY; - ObjectDeleter runQueueDeleter(sRunQueue); - - status_t result = sRunQueue->GetInitStatus(); - if (result != B_OK) - return result; - - ArrayDeleter cpuRunQueuesDeleter; - if (cpuCount > 1) { - sCPURunQueues = new(std::nothrow) SimpleRunQueue[cpuCount]; - if (sCPURunQueues == NULL) - return B_NO_MEMORY; - cpuRunQueuesDeleter.SetTo(sCPURunQueues); - - for (int i = 0; i < cpuCount; i++) { - result = sCPURunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - } - - gScheduler = &kSimpleOps; - - add_debugger_command_etc("run_queue", &dump_run_queue, - "List threads in run queue", "\nLists threads in run queue", 0); - add_debugger_command_etc("cpu_heap", &dump_cpu_heap, - "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", - 0); - - cpuHeapDeleter.Detach(); - cpuEntriesDeleter.Detach(); - runQueueDeleter.Detach(); - cpuRunQueuesDeleter.Detach(); - return B_OK; -} diff --git a/src/system/kernel/scheduler/scheduler_simple.h b/src/system/kernel/scheduler/scheduler_simple.h deleted file mode 100644 index aba7f5afbd..0000000000 --- a/src/system/kernel/scheduler/scheduler_simple.h +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright 2008, Ingo Weinhold, ingo_weinhold@gmx.de. - * Distributed under the terms of the MIT License. - */ -#ifndef KERNEL_SCHEDULER_SIMPLE_H -#define KERNEL_SCHEDULER_SIMPLE_H - - -status_t scheduler_simple_init(); - - -#endif // KERNEL_SCHEDULER_SIMPLE_H diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index d13fec1a5b..210492d15e 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1784,10 +1784,8 @@ _dump_thread_info(Thread *thread, bool shortInfo) kprintf("flags: 0x%" B_PRIx32 "\n", thread->flags); kprintf("architecture dependant section:\n"); arch_thread_dump_info(&thread->arch_info); - if (gScheduler->dump_thread_data != NULL) { - kprintf("scheduler data:\n"); - gScheduler->dump_thread_data(thread); - } + kprintf("scheduler data:\n"); + scheduler_dump_thread_data(thread); } From 9d7e2acf34add493940b0a0bbd0986d018033201 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 24 Oct 2013 02:44:47 +0200 Subject: [PATCH 075/273] scheduler: Update scheduler_set_operation_mode() --- src/system/kernel/scheduler/scheduler.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 18357d26fe..a99e3276de 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1402,18 +1402,18 @@ scheduler_start(void) } -static status_t -set_operation_mode(scheduler_mode mode) +status_t +scheduler_set_operation_mode(scheduler_mode mode) { if (mode != SCHEDULER_MODE_PERFORMANCE && mode != SCHEDULER_MODE_POWER_SAVING) { return B_BAD_VALUE; } -#ifdef TRACE_SCHEDULER - const char* modeNames = { "performance", "power saving" }; -#endif - TRACE("switching scheduler to %s mode\n", modeNames[mode]); + const char* modeNames[] = { "performance", "power saving" }; + dprintf("scheduler: switching to %s mode\n", modeNames[mode]); + + InterruptsSpinLocker _(gSchedulerLock); sSchedulerMode = mode; switch (mode) { @@ -1627,9 +1627,9 @@ _scheduler_init() } #if 1 - set_operation_mode(SCHEDULER_MODE_POWER_SAVING); + scheduler_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); #else - set_operation_mode(SCHEDULER_MODE_PERFORMANCE); + scheduler_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); #endif add_debugger_command_etc("run_queue", &dump_run_queue, From 45ff5300693f788a134f62eaa5f5982413cd10ec Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 25 Oct 2013 02:35:07 +0200 Subject: [PATCH 076/273] scheduler: Be more demanding when cancelling penalties --- src/system/kernel/scheduler/scheduler.cpp | 34 ++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index a99e3276de..4ed0f5218d 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -82,6 +82,7 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { int32 fCoreID; + bigtime_t fReachedBottom; bigtime_t fActiveTime; int32 fCPUBoundThreads; @@ -604,22 +605,8 @@ cancel_penalty(Thread* thread) if (schedulerThreadData->priority_penalty != 0) TRACE("cancelling thread %ld penalty\n", thread->id); - switch (sSchedulerMode) { - case SCHEDULER_MODE_PERFORMANCE: - schedulerThreadData->additional_penalty = 0; - schedulerThreadData->priority_penalty = 0; - break; - - case SCHEDULER_MODE_POWER_SAVING: - if (schedulerThreadData->additional_penalty != 0) - schedulerThreadData->additional_penalty /= 2; - else if (schedulerThreadData->priority_penalty != 0) - schedulerThreadData->priority_penalty--; - break; - - default: - break; - } + schedulerThreadData->additional_penalty = 0; + schedulerThreadData->priority_penalty = 0; } @@ -924,9 +911,13 @@ enqueue(Thread* thread, bool newOne) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - bigtime_t hasSlept = system_time() - schedulerThreadData->went_sleep; - if (newOne && hasSlept > kThreadQuantum) - cancel_penalty(thread); + int32 core = schedulerThreadData->previous_core; + if (newOne && core >= 0) { + if (schedulerThreadData->went_sleep + kThreadQuantum / 4 + < sCoreEntries[core].fReachedBottom) { + cancel_penalty(thread); + } + } int32 threadPriority = get_effective_priority(thread); @@ -1225,6 +1216,11 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) sCoreEntries[thisCore].fActiveTime += active; } + if (thread_is_idle_thread(nextThread) + || get_effective_priority(nextThread) == B_LOWEST_ACTIVE_PRIORITY) { + sCoreEntries[thisCore].fReachedBottom = system_time(); + } + if (!thread_is_idle_thread(nextThread)) { oldThread->cpu->last_kernel_time = nextThread->kernel_time; oldThread->cpu->last_user_time = nextThread->user_time; From 1df2e75540af9a6331fcf0914719c4b3d71ee551 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 27 Oct 2013 18:14:48 +0100 Subject: [PATCH 077/273] scheduler: Increase penalty of waiting threads The fact that thread is waiting doesn't mean that it is nice to the others. If the thread, indeed, waits for a longer time its penalty will be cancelled anyway, however if the thread waits for a very short time do not count that as being nice since lower priority threads didn't have much chance to run. --- src/system/kernel/scheduler/scheduler.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 4ed0f5218d..9557acc910 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -728,7 +728,7 @@ is_task_small(Thread* thread) { int32 priority = get_effective_priority(thread); int32 penalty = thread->scheduler_data->priority_penalty; - return penalty == 0 || priority >= B_DISPLAY_PRIORITY; + return penalty < 2 || priority >= B_DISPLAY_PRIORITY; } @@ -1244,10 +1244,8 @@ _scheduler_reschedule(void) // update CPU heap so that old thread would have CPU properly chosen Thread* nextThread = sRunQueues[thisCore].PeekMaximum(); - if (nextThread != NULL) { - update_priority_heaps(thisCPU, - get_effective_priority(nextThread)); - } + if (nextThread != NULL) + update_priority_heaps(thisCPU, get_effective_priority(nextThread)); switch (oldThread->next_state) { case B_THREAD_RUNNING: @@ -1271,6 +1269,7 @@ _scheduler_reschedule(void) break; case B_THREAD_SUSPENDED: + increase_penalty(oldThread); thread_goes_away(oldThread); TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; @@ -1278,6 +1277,7 @@ _scheduler_reschedule(void) thread_goes_away(oldThread); break; default: + increase_penalty(oldThread); thread_goes_away(oldThread); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, oldThread->next_state); From 890ba7415c79c371440981da45a96b3b3476b414 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 27 Oct 2013 20:05:20 +0100 Subject: [PATCH 078/273] scheduler: Decide whether to cancel thread penalty --- src/system/kernel/scheduler/scheduler.cpp | 102 ++++++++++++++++++++-- 1 file changed, 94 insertions(+), 8 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 9557acc910..49a241d66e 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -52,6 +52,8 @@ const bigtime_t kThreadQuantum = 1000; const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; +const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; + const bigtime_t kCacheExpire = 100000; static int sDisableSmallTaskPacking; @@ -82,7 +84,11 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { int32 fCoreID; + bigtime_t fStartedBottom; bigtime_t fReachedBottom; + bigtime_t fStartedIdle; + bigtime_t fReachedIdle; + bigtime_t fActiveTime; int32 fCPUBoundThreads; @@ -902,6 +908,51 @@ thread_goes_away(Thread* thread) } +static inline bool +should_cancel_penalty(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + int32 core = schedulerThreadData->previous_core; + + if (core < -1) + return false; + + bigtime_t now = system_time(); + bigtime_t wentSleep = schedulerThreadData->went_sleep; + + if (wentSleep < sCoreEntries[core].fReachedIdle) + return true; + + if (sCoreEntries[core].fStartedIdle != 0) { + if (wentSleep < sCoreEntries[core].fStartedIdle + && now - sCoreEntries[core].fStartedIdle >= kMinimalWaitTime) { + return true; + } + + if (wentSleep - sCoreEntries[core].fStartedIdle >= kMinimalWaitTime) + return true; + } + + if (get_effective_priority(thread) == B_LOWEST_ACTIVE_PRIORITY) + return false; + + if (wentSleep < sCoreEntries[core].fReachedIdle) + return true; + + if (sCoreEntries[core].fStartedBottom != 0) { + if (wentSleep < sCoreEntries[core].fStartedBottom + && now - sCoreEntries[core].fStartedBottom >= kMinimalWaitTime) { + return true; + } + + if (wentSleep - sCoreEntries[core].fStartedBottom >= kMinimalWaitTime) + return true; + } + + return false; +} + + static void enqueue(Thread* thread, bool newOne) { @@ -913,9 +964,18 @@ enqueue(Thread* thread, bool newOne) int32 core = schedulerThreadData->previous_core; if (newOne && core >= 0) { - if (schedulerThreadData->went_sleep + kThreadQuantum / 4 - < sCoreEntries[core].fReachedBottom) { - cancel_penalty(thread); + int32 priority = get_effective_priority(thread); + + if (priority == B_LOWEST_ACTIVE_PRIORITY) { + if (schedulerThreadData->went_sleep + < sCoreEntries[core].fReachedIdle) { + cancel_penalty(thread); + } + } else { + if (schedulerThreadData->went_sleep + < sCoreEntries[core].fReachedBottom) { + cancel_penalty(thread); + } } } @@ -1207,6 +1267,19 @@ dequeue_thread(int32 thisCPU) static inline void track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) { + bigtime_t now = system_time(); + bigtime_t usedTime = now - oldThread->scheduler_data->quantum_start; + + if (thread_is_idle_thread(oldThread) && usedTime >= kMinimalWaitTime) { + sCoreEntries[thisCore].fReachedBottom = now - kMinimalWaitTime; + sCoreEntries[thisCore].fReachedIdle = now - kMinimalWaitTime; + } + + if (get_effective_priority(oldThread) == B_LOWEST_ACTIVE_PRIORITY + && usedTime >= kMinimalWaitTime) { + sCoreEntries[thisCore].fReachedBottom = now - kMinimalWaitTime; + } + if (!thread_is_idle_thread(oldThread)) { bigtime_t active = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) @@ -1216,9 +1289,21 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) sCoreEntries[thisCore].fActiveTime += active; } - if (thread_is_idle_thread(nextThread) - || get_effective_priority(nextThread) == B_LOWEST_ACTIVE_PRIORITY) { - sCoreEntries[thisCore].fReachedBottom = system_time(); + int32 oldPriority = get_effective_priority(oldThread); + int32 nextPriority = get_effective_priority(nextThread); + + if (thread_is_idle_thread(nextThread)) { + if (!thread_is_idle_thread(oldThread)) + sCoreEntries[thisCore].fStartedIdle = now; + if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) + sCoreEntries[thisCore].fStartedBottom = now; + } else if (nextPriority == B_LOWEST_ACTIVE_PRIORITY) { + sCoreEntries[thisCore].fStartedIdle = 0; + if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) + sCoreEntries[thisCore].fStartedBottom = now; + } else { + sCoreEntries[thisCore].fStartedBottom = 0; + sCoreEntries[thisCore].fStartedIdle = 0; } if (!thread_is_idle_thread(nextThread)) { @@ -1337,7 +1422,8 @@ _scheduler_reschedule(void) bigtime_t quantum = compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); - } + } else + nextThread->scheduler_data->quantum_start = system_time(); if (nextThread != oldThread) scheduler_switch_thread(oldThread, nextThread); @@ -1622,7 +1708,7 @@ _scheduler_init() return result; } -#if 1 +#if 0 scheduler_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); #else scheduler_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); From d80cdf504fe9db6f6f7be62fd456ccdae4ea9178 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 27 Oct 2013 22:39:56 +0100 Subject: [PATCH 079/273] scheduler: Keep track of core and logical CPU load --- src/system/kernel/scheduler/scheduler.cpp | 69 +++++++++++++++++++++-- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 49a241d66e..a11e191957 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -73,6 +73,11 @@ static int32 (*sChooseCore)(Thread* thread); // the higher priority threads. struct CPUEntry : public MinMaxHeapLinkImpl { int32 fCPUNumber; + + bigtime_t fMeasureActiveTime; + bigtime_t fMeasureTime; + + int fLoad; }; typedef MinMaxHeap CPUHeap; static CPUEntry* sCPUEntries; @@ -93,6 +98,8 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { int32 fCPUBoundThreads; int32 fThreads; + + int fLoad; }; static CoreEntry* sCoreEntries; @@ -283,13 +290,13 @@ dump_heap(CPUHeap* heap) { CPUHeap temp(smp_get_num_cpus()); - kprintf("cpu priority actual priority\n"); + kprintf("cpu priority load\n"); CPUEntry* entry = heap->PeekMinimum(); while (entry) { int32 cpu = entry->fCPUNumber; int32 key = CPUHeap::GetKey(entry); - kprintf("%3" B_PRId32 " %8" B_PRId32 " %15" B_PRId32 "\n", cpu, key, - get_effective_priority(gCPU[cpu].running_thread)); + kprintf("%3" B_PRId32 " %8" B_PRId32 " %3d%%\n", cpu, key, + sCPUEntries[cpu].fLoad / 10); heap->RemoveMinimum(); temp.Insert(entry, key); @@ -311,12 +318,14 @@ static void dump_core_thread_heap(CoreThreadHeap* heap) { CoreThreadHeap temp(sRunQueueCount); + int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; CoreEntry* entry = heap->PeekMinimum(); while (entry) { int32 key = CoreThreadHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 "\n", - entry->fCoreID, key, entry->fThreads, entry->fCPUBoundThreads); + kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 + " %3d%%\n", entry->fCoreID, key, entry->fThreads, + entry->fCPUBoundThreads, entry->fLoad / cpuPerCore / 10); heap->RemoveMinimum(); temp.Insert(entry, key); @@ -364,7 +373,7 @@ dump_cpu_heap(int argc, char** argv) entry = temp.PeekRoot(); } - kprintf("\ncore key threads cpu-bound\n"); + kprintf("\ncore key threads cpu-bound load\n"); dump_core_thread_heap(sCoreThreadHeap); dump_core_thread_heap(sCoreCPUBoundThreadHeap); @@ -1264,6 +1273,45 @@ dequeue_thread(int32 thisCPU) } +static inline void +compute_cpu_load(int32 cpu) +{ + const bigtime_t kLoadMeasureInterval = 50000; + const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; + + int32 thisCPU = smp_get_current_cpu(); + + bigtime_t now = system_time(); + bigtime_t deltaTime = now - sCPUEntries[cpu].fMeasureTime; + + if (deltaTime < kLoadMeasureInterval) + return; + + int oldLoad = sCPUEntries[cpu].fLoad; + + int load = sCPUEntries[cpu].fMeasureActiveTime * 1000; + load /= max_c(now - sCPUEntries[cpu].fMeasureTime, 1); + + sCPUEntries[cpu].fMeasureActiveTime = 0; + sCPUEntries[cpu].fMeasureTime = now; + + deltaTime += kIntervalInaccuracy; + int n = max_c(deltaTime / kLoadMeasureInterval, 1); + if (n > 10) + sCPUEntries[cpu].fLoad = load; + else { + load *= (1 << n) - 1; + sCPUEntries[cpu].fLoad = (sCPUEntries[cpu].fLoad + load) / (1 << n); + } + + if (oldLoad != load) { + int32 core = sCPUToCore[cpu]; + sCoreEntries[core].fLoad -= oldLoad; + sCoreEntries[core].fLoad += sCPUEntries[cpu].fLoad; + } +} + + static inline void track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) { @@ -1286,9 +1334,12 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) + (oldThread->user_time - oldThread->cpu->last_user_time); atomic_add64(&oldThread->cpu->active_time, active); + sCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; sCoreEntries[thisCore].fActiveTime += active; } + compute_cpu_load(smp_get_current_cpu()); + int32 oldPriority = get_effective_priority(oldThread); int32 nextPriority = get_effective_priority(nextThread); @@ -1650,6 +1701,7 @@ _scheduler_init() sCoreEntries[i].fActiveTime = 0; sCoreEntries[i].fThreads = 0; sCoreEntries[i].fCPUBoundThreads = 0; + sCoreEntries[i].fLoad = 0; status_t result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); if (result != B_OK) @@ -1668,6 +1720,11 @@ _scheduler_init() for (int32 i = 0; i < cpuCount; i++) { sCPUEntries[i].fCPUNumber = i; + + sCPUEntries[i].fMeasureActiveTime = 0; + sCPUEntries[i].fMeasureTime = 0; + sCPUEntries[i].fLoad = 0; + int32 core = sCPUToCore[i]; int32 package = sCPUToPackage[i]; From dc38e6ca871a7da1b033fb0c027950e6ca53ab24 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 28 Oct 2013 00:39:16 +0100 Subject: [PATCH 080/273] scheduler: Use core load to distribute threads --- src/system/kernel/scheduler/scheduler.cpp | 220 ++++++++-------------- 1 file changed, 74 insertions(+), 146 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index a11e191957..1e55a93256 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -56,6 +56,8 @@ const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; const bigtime_t kCacheExpire = 100000; +const bigtime_t kHighLoad = 600; + static int sDisableSmallTaskPacking; static int32 sSmallTaskCore; @@ -85,7 +87,7 @@ static CPUHeap* sCPUPriorityHeaps; struct CoreEntry : public DoublyLinkedListLinkImpl { HeapLink fPriorityHeapLink; - MinMaxHeapLink fThreadHeapLink; + MinMaxHeapLink fLoadHeapLink; int32 fCoreID; @@ -96,9 +98,6 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { bigtime_t fActiveTime; - int32 fCPUBoundThreads; - int32 fThreads; - int fLoad; }; @@ -108,14 +107,11 @@ typedef Heap, CorePriorityHeap; static CorePriorityHeap* sCorePriorityHeap; -typedef MinMaxHeap, - MinMaxHeapMemberGetLink > - CoreThreadHeap; -static CoreThreadHeap* sCoreThreadHeap; -static CoreThreadHeap* sCoreCPUBoundThreadHeap; - -static int32 sCPUBoundThreads; -static int32 sAssignedThreads; +typedef MinMaxHeap, + MinMaxHeapMemberGetLink > + CoreLoadHeap; +static CoreLoadHeap* sCoreLoadHeap; +static CoreLoadHeap* sCoreHighLoadHeap; // sPackageUsageHeap is used to decide which core should be woken up from the // idle state. When aiming for performance we should use as many packages as @@ -201,7 +197,7 @@ scheduler_thread_data::Init() static inline int get_minimal_priority(Thread* thread) { - return min_c(thread->priority, 25) / 5; + return max_c(min_c(thread->priority, 25) / 5, 1); } @@ -315,17 +311,16 @@ dump_heap(CPUHeap* heap) static void -dump_core_thread_heap(CoreThreadHeap* heap) +dump_core_load_heap(CoreLoadHeap* heap) { - CoreThreadHeap temp(sRunQueueCount); + CoreLoadHeap temp(sRunQueueCount); int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; CoreEntry* entry = heap->PeekMinimum(); while (entry) { - int32 key = CoreThreadHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %6" B_PRId32 " %7" B_PRId32 " %9" B_PRId32 - " %3d%%\n", entry->fCoreID, key, entry->fThreads, - entry->fCPUBoundThreads, entry->fLoad / cpuPerCore / 10); + int key = CoreLoadHeap::GetKey(entry); + kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, + entry->fLoad / cpuPerCore / 10); heap->RemoveMinimum(); temp.Insert(entry, key); @@ -335,7 +330,7 @@ dump_core_thread_heap(CoreThreadHeap* heap) entry = temp.PeekMinimum(); while (entry) { - int32 key = CoreThreadHeap::GetKey(entry); + int key = CoreLoadHeap::GetKey(entry); temp.RemoveMinimum(); heap->Insert(entry, key); entry = temp.PeekMinimum(); @@ -373,9 +368,10 @@ dump_cpu_heap(int argc, char** argv) entry = temp.PeekRoot(); } - kprintf("\ncore key threads cpu-bound load\n"); - dump_core_thread_heap(sCoreThreadHeap); - dump_core_thread_heap(sCoreCPUBoundThreadHeap); + kprintf("\ncore load\n"); + dump_core_load_heap(sCoreLoadHeap); + kprintf("---------\n"); + dump_core_load_heap(sCoreHighLoadHeap); for (int32 i = 0; i < sRunQueueCount; i++) { kprintf("\nCore %" B_PRId32 " heap:\n", i); @@ -517,43 +513,40 @@ scheduler_dump_thread_data(Thread* thread) static void -update_thread_heaps(int32 core) +update_load_heaps(int32 core) { ASSERT(!sSingleCore); CoreEntry* entry = &sCoreEntries[core]; - ASSERT(entry->fCPUBoundThreads >= 0 - && entry->fCPUBoundThreads <= entry->fThreads); - ASSERT(entry->fThreads >= 0 - && entry->fThreads <= thread_max_threads()); + int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; + int newKey = entry->fLoad / cpuPerCore; + int oldKey = CoreLoadHeap::GetKey(entry); - int32 newKey = entry->fCPUBoundThreads * thread_max_threads(); - newKey += entry->fThreads; - - int32 oldKey = CoreThreadHeap::GetKey(entry); + ASSERT(oldKey >= 0 && oldKey <= 1000); + ASSERT(newKey >= 0 && newKey <= 1000); if (oldKey == newKey) return; - if (newKey > thread_max_threads()) { - if (oldKey <= thread_max_threads()) { - sCoreThreadHeap->ModifyKey(entry, -1); - ASSERT(sCoreThreadHeap->PeekMinimum() == entry); - sCoreThreadHeap->RemoveMinimum(); + if (newKey > kHighLoad) { + if (oldKey <= kHighLoad) { + sCoreLoadHeap->ModifyKey(entry, -1); + ASSERT(sCoreLoadHeap->PeekMinimum() == entry); + sCoreLoadHeap->RemoveMinimum(); - sCoreCPUBoundThreadHeap->Insert(entry, newKey); + sCoreHighLoadHeap->Insert(entry, newKey); } else - sCoreCPUBoundThreadHeap->ModifyKey(entry, newKey); + sCoreHighLoadHeap->ModifyKey(entry, newKey); } else { - if (oldKey > thread_max_threads()) { - sCoreCPUBoundThreadHeap->ModifyKey(entry, -1); - ASSERT(sCoreCPUBoundThreadHeap->PeekMinimum() == entry); - sCoreCPUBoundThreadHeap->RemoveMinimum(); + if (oldKey > kHighLoad) { + sCoreHighLoadHeap->ModifyKey(entry, -1); + ASSERT(sCoreHighLoadHeap->PeekMinimum() == entry); + sCoreHighLoadHeap->RemoveMinimum(); - sCoreThreadHeap->Insert(entry, newKey); + sCoreLoadHeap->Insert(entry, newKey); } else - sCoreThreadHeap->ModifyKey(entry, newKey); + sCoreLoadHeap->ModifyKey(entry, newKey); } } @@ -566,8 +559,8 @@ disable_small_task_packing(void) ASSERT(sDisableSmallTaskPacking == 0); ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); - ASSERT(sAssignedThreads > 0); - sDisableSmallTaskPacking = sAssignedThreads * 64; +// ASSERT(sAssignedThreads > 0); +// sDisableSmallTaskPacking = sAssignedThreads * 64; sSmallTaskCore = -1; } @@ -575,7 +568,7 @@ disable_small_task_packing(void) static inline void increase_penalty(Thread* thread) { - if (thread->priority <= B_LOWEST_ACTIVE_PRIORITY) + if (thread->priority < B_LOWEST_ACTIVE_PRIORITY) return; if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) return; @@ -589,23 +582,6 @@ increase_penalty(Thread* thread) const int kMinimalPriority = get_minimal_priority(thread); if (thread->priority - oldPenalty <= kMinimalPriority) { - int32 core = schedulerThreadData->previous_core; - ASSERT(core >= 0); - - int32 additionalPenalty = schedulerThreadData->additional_penalty; - if (additionalPenalty == 0 && !sSingleCore) { - sCPUBoundThreads++; - sCoreEntries[core].fCPUBoundThreads++; - - update_thread_heaps(core); - } - - const int kSmallTaskThreshold = 50; - if (additionalPenalty > kSmallTaskThreshold && !sSingleCore) { - if (sSmallTaskCore == core) - disable_small_task_packing(); - } - schedulerThreadData->priority_penalty = oldPenalty; schedulerThreadData->additional_penalty++; } @@ -727,9 +703,9 @@ choose_core_performance(Thread* thread) int32 priority = get_effective_priority(thread); if (CorePriorityHeap::GetKey(entry) >= priority) { - entry = sCoreThreadHeap->PeekMinimum(); + entry = sCoreLoadHeap->PeekMinimum(); if (entry == NULL) - entry = sCoreCPUBoundThreadHeap->PeekMinimum(); + entry = sCoreHighLoadHeap->PeekMinimum(); } } @@ -758,10 +734,10 @@ choose_core_power_saving(Thread* thread) sDisableSmallTaskPacking--; if (!sDisableSmallTaskPacking && is_task_small(thread) - && sCoreThreadHeap->PeekMaximum() != NULL) { + && sCoreLoadHeap->PeekMaximum() != NULL) { // try to pack all threads on one core if (sSmallTaskCore < 0) - sSmallTaskCore = sCoreThreadHeap->PeekMaximum()->fCoreID; + sSmallTaskCore = sCoreLoadHeap->PeekMaximum()->fCoreID; entry = &sCoreEntries[sSmallTaskCore]; } else if (sCorePriorityHeap->PeekRoot() != NULL && CorePriorityHeap::GetKey(sCorePriorityHeap->PeekRoot()) @@ -778,9 +754,9 @@ choose_core_power_saving(Thread* thread) entry = package->fIdleCores.Last(); } else { // no idle cores, use least occupied core - entry = sCoreThreadHeap->PeekMinimum(); + entry = sCoreLoadHeap->PeekMinimum(); if (entry == NULL) - entry = sCoreCPUBoundThreadHeap->PeekMinimum(); + entry = sCoreHighLoadHeap->PeekMinimum(); } ASSERT(entry != NULL); @@ -808,6 +784,7 @@ choose_cpu(int32 core) static bool should_rebalance(Thread* thread) { +#if 0 ASSERT(!sSingleCore); if (thread_is_idle_thread(thread)) @@ -857,31 +834,11 @@ should_rebalance(Thread* thread) // won't get much worse... const int32 kBalanceThreshold = 3; return threadsAboveAverage > kBalanceThreshold; +#endif + return false; } -static void -assign_active_thread_to_core(Thread* thread) -{ - if (thread_is_idle_thread(thread) || sSingleCore) - return; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - ASSERT(schedulerThreadData->previous_core >= 0); - int32 core = schedulerThreadData->previous_core; - - sCoreEntries[core].fThreads++; - sAssignedThreads++; - - if (schedulerThreadData->additional_penalty != 0) { - sCoreEntries[core].fCPUBoundThreads++; - sCPUBoundThreads++; - } - - update_thread_heaps(core); -} - static inline void thread_goes_away(Thread* thread) @@ -896,24 +853,6 @@ thread_goes_away(Thread* thread) schedulerThreadData->went_sleep = system_time(); schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; - - if (sSingleCore) - return; - - ASSERT(sCoreEntries[core].fThreads > 0); - ASSERT(sCoreEntries[core].fThreads > sCoreEntries[core].fCPUBoundThreads - || (sCoreEntries[core].fThreads == sCoreEntries[core].fCPUBoundThreads - && schedulerThreadData->additional_penalty != 0)); - sCoreEntries[core].fThreads--; - sAssignedThreads--; - - if (schedulerThreadData->additional_penalty != 0) { - ASSERT(sCoreEntries[core].fCPUBoundThreads > 0); - sCoreEntries[core].fCPUBoundThreads--; - sCPUBoundThreads--; - } - - update_thread_heaps(core); } @@ -999,9 +938,6 @@ enqueue(Thread* thread, bool newOne) targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; ASSERT(targetCore == schedulerThreadData->previous_core); - - if (newOne) - assign_active_thread_to_core(thread); } else if (sSingleCore) { targetCore = 0; targetCPU = choose_cpu(targetCore); @@ -1015,20 +951,14 @@ enqueue(Thread* thread, bool newOne) targetCPU = thread->previous_cpu->cpu_num; targetCore = sCPUToCore[targetCPU]; } else { - if (!newOne) - thread_goes_away(thread); - targetCore = choose_core(thread); targetCPU = choose_cpu(targetCore); } schedulerThreadData->previous_core = targetCore; - assign_active_thread_to_core(thread); } else { targetCore = schedulerThreadData->previous_core; targetCPU = choose_cpu(targetCore); - if (newOne) - assign_active_thread_to_core(thread); } ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); @@ -1116,17 +1046,12 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->id, priority, thread->priority, get_effective_priority(thread)); - if (thread->state == B_THREAD_RUNNING) - thread_goes_away(thread); - if (thread->state != B_THREAD_READY) { cancel_penalty(thread); thread->priority = priority; - if (thread->state == B_THREAD_RUNNING) { - assign_active_thread_to_core(thread); + if (thread->state == B_THREAD_RUNNING) update_priority_heaps(thread->cpu->cpu_num, priority); - } return; } @@ -1143,7 +1068,6 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) int32 previousCore = thread->scheduler_data->previous_core; ASSERT(previousCore >= 0); sRunQueues[previousCore].Remove(thread); - thread_goes_away(thread); // set priority and re-insert cancel_penalty(thread); @@ -1276,11 +1200,11 @@ dequeue_thread(int32 thisCPU) static inline void compute_cpu_load(int32 cpu) { + ASSERT(!sSingleCore); + const bigtime_t kLoadMeasureInterval = 50000; const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; - int32 thisCPU = smp_get_current_cpu(); - bigtime_t now = system_time(); bigtime_t deltaTime = now - sCPUEntries[cpu].fMeasureTime; @@ -1288,26 +1212,34 @@ compute_cpu_load(int32 cpu) return; int oldLoad = sCPUEntries[cpu].fLoad; + ASSERT(oldLoad >= 0 && oldLoad <= 1000); int load = sCPUEntries[cpu].fMeasureActiveTime * 1000; load /= max_c(now - sCPUEntries[cpu].fMeasureTime, 1); + load = max_c(min_c(load, 1000), 0); sCPUEntries[cpu].fMeasureActiveTime = 0; sCPUEntries[cpu].fMeasureTime = now; deltaTime += kIntervalInaccuracy; - int n = max_c(deltaTime / kLoadMeasureInterval, 1); + int n = deltaTime / kLoadMeasureInterval; + ASSERT(n > 0); + if (n > 10) sCPUEntries[cpu].fLoad = load; else { load *= (1 << n) - 1; sCPUEntries[cpu].fLoad = (sCPUEntries[cpu].fLoad + load) / (1 << n); + ASSERT(sCPUEntries[cpu].fLoad >= 0 && sCPUEntries[cpu].fLoad <= 1000); } if (oldLoad != load) { int32 core = sCPUToCore[cpu]; + sCoreEntries[core].fLoad -= oldLoad; sCoreEntries[core].fLoad += sCPUEntries[cpu].fLoad; + + update_load_heaps(core); } } @@ -1338,7 +1270,8 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) sCoreEntries[thisCore].fActiveTime += active; } - compute_cpu_load(smp_get_current_cpu()); + if (!sSingleCore) + compute_cpu_load(smp_get_current_cpu()); int32 oldPriority = get_effective_priority(oldThread); int32 nextPriority = get_effective_priority(nextThread); @@ -1410,7 +1343,6 @@ _scheduler_reschedule(void) TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: - thread_goes_away(oldThread); break; default: increase_penalty(oldThread); @@ -1455,7 +1387,6 @@ _scheduler_reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; ASSERT(nextThread->scheduler_data->previous_core == thisCore); - //nextThread->scheduler_data->previous_core = thisCore; // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); @@ -1685,25 +1616,22 @@ _scheduler_init() return B_NO_MEMORY; ObjectDeleter corePriorityHeapDeleter(sCorePriorityHeap); - sCoreThreadHeap = new CoreThreadHeap; - if (sCoreThreadHeap == NULL) + sCoreLoadHeap = new CoreLoadHeap; + if (sCoreLoadHeap == NULL) return B_NO_MEMORY; - ObjectDeleter coreThreadHeapDeleter(sCoreThreadHeap); + ObjectDeleter coreLoadHeapDeleter(sCoreLoadHeap); - sCoreCPUBoundThreadHeap = new CoreThreadHeap(coreCount); - if (sCoreCPUBoundThreadHeap == NULL) + sCoreHighLoadHeap = new CoreLoadHeap(coreCount); + if (sCoreHighLoadHeap == NULL) return B_NO_MEMORY; - ObjectDeleter coreCPUThreadHeapDeleter( - sCoreCPUBoundThreadHeap); + ObjectDeleter coreHighLoadHeapDeleter(sCoreHighLoadHeap); for (int32 i = 0; i < coreCount; i++) { sCoreEntries[i].fCoreID = i; sCoreEntries[i].fActiveTime = 0; - sCoreEntries[i].fThreads = 0; - sCoreEntries[i].fCPUBoundThreads = 0; sCoreEntries[i].fLoad = 0; - status_t result = sCoreThreadHeap->Insert(&sCoreEntries[i], 0); + status_t result = sCoreLoadHeap->Insert(&sCoreEntries[i], 0); if (result != B_OK) return result; @@ -1765,7 +1693,7 @@ _scheduler_init() return result; } -#if 0 +#if 1 scheduler_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); #else scheduler_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); @@ -1783,8 +1711,8 @@ _scheduler_init() runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); - coreCPUThreadHeapDeleter.Detach(); - coreThreadHeapDeleter.Detach(); + coreHighLoadHeapDeleter.Detach(); + coreLoadHeapDeleter.Detach(); corePriorityHeapDeleter.Detach(); cpuPriorityHeapDeleter.Detach(); coreEntriesDeleter.Detach(); From 5e2701a2b50591b76e8981935ceea0e7e07573ea Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 28 Oct 2013 01:38:54 +0100 Subject: [PATCH 081/273] scheduler: Keep track of the load each thread produces --- src/system/kernel/scheduler/scheduler.cpp | 135 ++++++++++++++-------- 1 file changed, 88 insertions(+), 47 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 1e55a93256..b2bebabd8f 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -168,6 +168,10 @@ struct scheduler_thread_data { bigtime_t stolen_time; bigtime_t quantum_start; + bigtime_t measure_active_time; + bigtime_t measure_time; + int load; + bigtime_t went_sleep; bigtime_t went_sleep_active; @@ -184,6 +188,10 @@ scheduler_thread_data::Init() time_left = 0; stolen_time = 0; + measure_active_time = 0; + measure_time = 0; + load = 0; + went_sleep = 0; went_sleep_active = 0; @@ -499,6 +507,7 @@ scheduler_dump_thread_data(Thread* thread) additionalPenalty, schedulerThreadData->additional_penalty); kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", schedulerThreadData->stolen_time); + kprintf("\tload:\t\t\t%d%%\n", schedulerThreadData->load / 10); kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", schedulerThreadData->went_sleep); kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", @@ -839,6 +848,79 @@ should_rebalance(Thread* thread) } +static inline int +compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int& load) +{ + const bigtime_t kLoadMeasureInterval = 50000; + const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; + + bigtime_t now = system_time(); + + if (measureTime == 0) { + measureTime = now; + return -1; + } + + bigtime_t deltaTime = now - measureTime; + + if (deltaTime < kLoadMeasureInterval) + return -1; + + int oldLoad = load; + ASSERT(oldLoad >= 0 && oldLoad <= 1000); + + int newLoad = measureActiveTime * 1000; + newLoad /= max_c(deltaTime, 1); + newLoad = max_c(min_c(newLoad, 1000), 0); + + measureActiveTime = 0; + measureTime = now; + + deltaTime += kIntervalInaccuracy; + int n = deltaTime / kLoadMeasureInterval; + ASSERT(n > 0); + + if (n > 10) + load = newLoad; + else { + newLoad *= (1 << n) - 1; + load = (load + newLoad) / (1 << n); + ASSERT(load >= 0 && load <= 1000); + } + + return oldLoad; +} + + +static inline void +compute_cpu_load(int32 cpu) +{ + ASSERT(!sSingleCore); + + int oldLoad = compute_load(sCPUEntries[cpu].fMeasureTime, + sCPUEntries[cpu].fMeasureActiveTime, sCPUEntries[cpu].fLoad); + if (oldLoad < 0) + return; + + if (oldLoad != sCPUEntries[cpu].fLoad) { + int32 core = sCPUToCore[cpu]; + + sCoreEntries[core].fLoad -= oldLoad; + sCoreEntries[core].fLoad += sCPUEntries[cpu].fLoad; + + update_load_heaps(core); + } +} + + +static inline void +compute_thread_load(Thread* thread) +{ + compute_load(thread->scheduler_data->measure_time, + thread->scheduler_data->measure_active_time, + thread->scheduler_data->load); +} + static inline void thread_goes_away(Thread* thread) @@ -908,6 +990,8 @@ enqueue(Thread* thread, bool newOne) thread->state = thread->next_state = B_THREAD_READY; + compute_thread_load(thread); + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; int32 core = schedulerThreadData->previous_core; @@ -1018,6 +1102,8 @@ scheduler_enqueue_in_run_queue(Thread *thread) static inline void put_back(Thread* thread) { + compute_thread_load(thread); + bool pinned = sPinnedRunQueues != NULL && thread->pinned_to_cpu > 0; if (pinned) { @@ -1197,53 +1283,6 @@ dequeue_thread(int32 thisCPU) } -static inline void -compute_cpu_load(int32 cpu) -{ - ASSERT(!sSingleCore); - - const bigtime_t kLoadMeasureInterval = 50000; - const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; - - bigtime_t now = system_time(); - bigtime_t deltaTime = now - sCPUEntries[cpu].fMeasureTime; - - if (deltaTime < kLoadMeasureInterval) - return; - - int oldLoad = sCPUEntries[cpu].fLoad; - ASSERT(oldLoad >= 0 && oldLoad <= 1000); - - int load = sCPUEntries[cpu].fMeasureActiveTime * 1000; - load /= max_c(now - sCPUEntries[cpu].fMeasureTime, 1); - load = max_c(min_c(load, 1000), 0); - - sCPUEntries[cpu].fMeasureActiveTime = 0; - sCPUEntries[cpu].fMeasureTime = now; - - deltaTime += kIntervalInaccuracy; - int n = deltaTime / kLoadMeasureInterval; - ASSERT(n > 0); - - if (n > 10) - sCPUEntries[cpu].fLoad = load; - else { - load *= (1 << n) - 1; - sCPUEntries[cpu].fLoad = (sCPUEntries[cpu].fLoad + load) / (1 << n); - ASSERT(sCPUEntries[cpu].fLoad >= 0 && sCPUEntries[cpu].fLoad <= 1000); - } - - if (oldLoad != load) { - int32 core = sCPUToCore[cpu]; - - sCoreEntries[core].fLoad -= oldLoad; - sCoreEntries[core].fLoad += sCPUEntries[cpu].fLoad; - - update_load_heaps(core); - } -} - - static inline void track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) { @@ -1266,6 +1305,7 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) + (oldThread->user_time - oldThread->cpu->last_user_time); atomic_add64(&oldThread->cpu->active_time, active); + oldThread->scheduler_data->measure_active_time += active; sCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; sCoreEntries[thisCore].fActiveTime += active; } @@ -1387,6 +1427,7 @@ _scheduler_reschedule(void) nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; ASSERT(nextThread->scheduler_data->previous_core == thisCore); + compute_thread_load(nextThread); // track kernel time (user time is tracked in thread_at_kernel_entry()) scheduler_update_thread_times(oldThread, nextThread); From 6d96f462dcea05a929f52e0c45aa5a2ff1efb9ae Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 28 Oct 2013 02:44:46 +0100 Subject: [PATCH 082/273] scheduler: Use load information to migrate threads --- src/system/kernel/scheduler/scheduler.cpp | 124 ++++++++++++++-------- 1 file changed, 79 insertions(+), 45 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index b2bebabd8f..8a07620315 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -57,8 +57,9 @@ const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; const bigtime_t kCacheExpire = 100000; const bigtime_t kHighLoad = 600; +const int kLoadDifference = 200; -static int sDisableSmallTaskPacking; +static bigtime_t sDisableSmallTaskPacking; static int32 sSmallTaskCore; static bool sSingleCore; @@ -66,6 +67,7 @@ static bool sSingleCore; static scheduler_mode sSchedulerMode; static int32 (*sChooseCore)(Thread* thread); +static bool (*sShouldRebalance)(Thread* thread); // Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical @@ -560,16 +562,24 @@ update_load_heaps(int32 core) } +static inline bool +is_small_task_packing_enabled(void) +{ + if (sDisableSmallTaskPacking == -1) + return false; + return sDisableSmallTaskPacking < system_time(); +} + + static inline void disable_small_task_packing(void) { ASSERT(!sSingleCore); - ASSERT(sDisableSmallTaskPacking == 0); + ASSERT(is_small_task_packing_enabled()); ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); -// ASSERT(sAssignedThreads > 0); -// sDisableSmallTaskPacking = sAssignedThreads * 64; + sDisableSmallTaskPacking = system_time() + kThreadQuantum * 100; sSmallTaskCore = -1; } @@ -726,9 +736,7 @@ choose_core_performance(Thread* thread) static inline bool is_task_small(Thread* thread) { - int32 priority = get_effective_priority(thread); - int32 penalty = thread->scheduler_data->priority_penalty; - return penalty < 2 || priority >= B_DISPLAY_PRIORITY; + return thread->scheduler_data->load <= 200; } @@ -739,10 +747,7 @@ choose_core_power_saving(Thread* thread) int32 priority = get_effective_priority(thread); - if (sDisableSmallTaskPacking > 0) - sDisableSmallTaskPacking--; - - if (!sDisableSmallTaskPacking && is_task_small(thread) + if (is_small_task_packing_enabled() && is_task_small(thread) && sCoreLoadHeap->PeekMaximum() != NULL) { // try to pack all threads on one core if (sSmallTaskCore < 0) @@ -791,9 +796,38 @@ choose_cpu(int32 core) static bool -should_rebalance(Thread* thread) +should_rebalance_performance(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; + + // If the thread produces more than 50% of the load, leave it here. In + // such situation it is better to move other threads away. + if (schedulerThreadData->load >= coreEntry->fLoad / 2) + return false; + + // If there is high load on this core but this thread does not contribute + // significantly consider giving it to someone less busy. + if (coreEntry->fLoad > kHighLoad) { + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); + if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference) + return true; + } + + // No cpu bound threads - the situation is quite good. Make sure it + // won't get much worse... + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); + if (other == NULL) + other = sCoreHighLoadHeap->PeekMinimum(); + return coreEntry->fLoad - other->fLoad >= kLoadDifference * 2; +} + + +static bool +should_rebalance_power_saving(Thread* thread) { -#if 0 ASSERT(!sSingleCore); if (thread_is_idle_thread(thread)) @@ -802,49 +836,47 @@ should_rebalance(Thread* thread) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; ASSERT(schedulerThreadData->previous_core >= 0); - CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; + int32 core = schedulerThreadData->previous_core; + CoreEntry* coreEntry = &sCoreEntries[core]; - // If this is a cpu bound thread and we have significantly more such threads - // than the average get rid of this one. - if (schedulerThreadData->additional_penalty != 0) { - int32 averageCPUBound = sCPUBoundThreads / sRunQueueCount; - return coreEntry->fCPUBoundThreads - averageCPUBound > 1; - } - - // If this thread is not cpu bound but we have at least one consider giving - // this one to someone less busy. - int32 averageThread = sAssignedThreads / sRunQueueCount; - if (coreEntry->fCPUBoundThreads > 0) { - CoreEntry* other = sCoreThreadHeap->PeekMinimum(); - if (other != NULL - && CoreThreadHeap::GetKey(other) <= averageThread) { - return true; - } - } - - int32 threadsAboveAverage = coreEntry->fThreads - averageThread; + // If the thread produces more than 50% of the load, leave it here. In + // such situation it is better to move other threads away. + // Unless we are trying to pack small tasks here, in such case get rid + // of CPU hungry thread and continue packing. + if (schedulerThreadData->load >= coreEntry->fLoad / 2) + return is_small_task_packing_enabled() && sSmallTaskCore == core; // All cores try to give us small tasks, check whether we have enough. - const int kSmallTaskCountThreshold = 5; - if (sDisableSmallTaskPacking == 0 && sSmallTaskCore == coreEntry->fCoreID) { - if (threadsAboveAverage > kSmallTaskCountThreshold) { + if (is_small_task_packing_enabled() && sSmallTaskCore == core) { + if (coreEntry->fLoad > kHighLoad) { if (!is_task_small(thread)) return true; - } else if (threadsAboveAverage > 2 * kSmallTaskCountThreshold) { + } else if (coreEntry->fLoad > (kHighLoad + 1000) / 2) disable_small_task_packing(); - } } - // Try our luck at small task packing. - if (sDisableSmallTaskPacking == 0 && is_task_small(thread)) - return sSmallTaskCore != coreEntry->fCoreID; + // Try small task packing. + if (is_small_task_packing_enabled() && is_task_small(thread)) + return sSmallTaskCore != core; // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... - const int32 kBalanceThreshold = 3; - return threadsAboveAverage > kBalanceThreshold; -#endif - return false; + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); + if (other == NULL) + other = sCoreHighLoadHeap->PeekMinimum(); + return coreEntry->fLoad - other->fLoad >= kLoadDifference; +} + + +static bool +should_rebalance(Thread* thread) +{ + ASSERT(!sSingleCore); + + if (thread_is_idle_thread(thread)) + return false; + + return sShouldRebalance(thread); } @@ -1526,12 +1558,14 @@ scheduler_set_operation_mode(scheduler_mode mode) sDisableSmallTaskPacking = -1; sSmallTaskCore = -1; sChooseCore = choose_core_performance; + sShouldRebalance = should_rebalance_performance; break; case SCHEDULER_MODE_POWER_SAVING: sDisableSmallTaskPacking = 0; sSmallTaskCore = -1; sChooseCore = choose_core_power_saving; + sShouldRebalance = should_rebalance_power_saving; break; default: From 22d824826747c96e7649b84ab5c837dfff27cf97 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 30 Oct 2013 00:48:07 +0100 Subject: [PATCH 083/273] kernel: Add support and interface for cpufreq modules --- headers/os/drivers/cpufreq.h | 26 +++++++++++++++++ headers/private/kernel/cpu.h | 3 ++ src/system/kernel/cpu.cpp | 55 +++++++++++++++++++++++++++++++++++- 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 headers/os/drivers/cpufreq.h diff --git a/headers/os/drivers/cpufreq.h b/headers/os/drivers/cpufreq.h new file mode 100644 index 0000000000..7a4301feeb --- /dev/null +++ b/headers/os/drivers/cpufreq.h @@ -0,0 +1,26 @@ +/* + * Copyright 2013, Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + */ +#ifndef _CPUFREQ_H +#define _CPUFREQ_H + + +#include + + +#define CPUFREQ_MODULES_PREFIX "power/cpufreq" + + +const int kCPUPerformanceScaleMax = 1000; + +typedef struct cpufreq_module_info { + module_info info; + + status_t (*increase_performance)(int delta, bool allowBoost); + status_t (*decrease_performance)(int delta); +} cpufreq_module_info; + + +#endif // _CPUFREQ_H + diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index e833a944cd..bcd06aa59b 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -104,6 +104,9 @@ extern inline cpu_ent *get_cpu_struct(void) { return &gCPU[smp_get_current_cpu() status_t cpu_build_topology_tree(void); cpu_topology_node* get_cpu_topology(void); +status_t increase_cpu_performance(int delta, bool allowBoost); +status_t decrease_cpu_performance(int delta); + void _user_clear_caches(void *address, size_t length, uint32 flags); bool _user_cpu_enabled(int32 cpu); status_t _user_set_cpu_enabled(int32 cpu, bool enabled); diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index bacc8a3ec7..b98ad77bdd 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. * @@ -14,6 +15,8 @@ #include +#include + #include #include #include @@ -25,6 +28,8 @@ cpu_ent gCPU[MAX_BOOT_CPUS]; uint32 gCPUCacheLevelCount; static cpu_topology_node sCPUTopology; +static cpufreq_module_info* sCPUPerformanceModule; + static spinlock sSetCpuLock; @@ -49,10 +54,40 @@ cpu_init_post_vm(kernel_args *args) } +static void +load_cpufreq_module() +{ + void* cookie = open_module_list(CPUFREQ_MODULES_PREFIX); + + while (true) { + char name[B_FILE_NAME_LENGTH]; + size_t nameLength = sizeof(name); + + if (read_next_module_name(cookie, name, &nameLength) != B_OK) + break; + + if (get_module(name, (module_info**)&sCPUPerformanceModule) == B_OK) { + dprintf("found cpufreq module: %s\n", name); + break; + } + } + + close_module_list(cookie); + + if (sCPUPerformanceModule == NULL) + dprintf("no valid cpufreq module found\n"); +} + + status_t cpu_init_post_modules(kernel_args *args) { - return arch_cpu_init_post_modules(args); + status_t result = arch_cpu_init_post_modules(args); + if (result != B_OK) + return result; + + load_cpufreq_module(); + return B_OK; } @@ -194,6 +229,24 @@ get_cpu_topology(void) } +status_t +increase_cpu_performance(int delta, bool allowBoost) +{ + if (sCPUPerformanceModule != NULL) + return sCPUPerformanceModule->increase_performance(delta, allowBoost); + return B_NOT_SUPPORTED; +} + + +status_t +decrease_cpu_performance(int delta) +{ + if (sCPUPerformanceModule != NULL) + return sCPUPerformanceModule->decrease_performance(delta); + return B_NOT_SUPPORTED; +} + + // #pragma mark - From 1e3cf82d8584947cc10c0c9ac3aa23ccbea5ca4e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 30 Oct 2013 00:49:24 +0100 Subject: [PATCH 084/273] scheduler: Manage CPU performance --- src/system/kernel/scheduler/scheduler.cpp | 49 +++++++++++++++++++---- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 8a07620315..7a90a8f930 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -30,6 +30,8 @@ #include #include +#include + #include "RunQueue.h" #include "scheduler_common.h" #include "scheduler_tracing.h" @@ -56,7 +58,9 @@ const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; const bigtime_t kCacheExpire = 100000; -const bigtime_t kHighLoad = 600; +const int kTargetLoad = 550; +const int kHighLoad = 700; +const int kMaxLoad = 1000; const int kLoadDifference = 200; static bigtime_t sDisableSmallTaskPacking; @@ -534,8 +538,8 @@ update_load_heaps(int32 core) int newKey = entry->fLoad / cpuPerCore; int oldKey = CoreLoadHeap::GetKey(entry); - ASSERT(oldKey >= 0 && oldKey <= 1000); - ASSERT(newKey >= 0 && newKey <= 1000); + ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); + ASSERT(newKey >= 0 && newKey <= kMaxLoad); if (oldKey == newKey) return; @@ -851,7 +855,7 @@ should_rebalance_power_saving(Thread* thread) if (coreEntry->fLoad > kHighLoad) { if (!is_task_small(thread)) return true; - } else if (coreEntry->fLoad > (kHighLoad + 1000) / 2) + } else if (coreEntry->fLoad > (kHighLoad + kMaxLoad) / 2) disable_small_task_packing(); } @@ -899,11 +903,11 @@ compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int& load) return -1; int oldLoad = load; - ASSERT(oldLoad >= 0 && oldLoad <= 1000); + ASSERT(oldLoad >= 0 && oldLoad <= kMaxLoad); - int newLoad = measureActiveTime * 1000; + int newLoad = measureActiveTime * kMaxLoad; newLoad /= max_c(deltaTime, 1); - newLoad = max_c(min_c(newLoad, 1000), 0); + newLoad = max_c(min_c(newLoad, kMaxLoad), 0); measureActiveTime = 0; measureTime = now; @@ -917,7 +921,7 @@ compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int& load) else { newLoad *= (1 << n) - 1; load = (load + newLoad) / (1 << n); - ASSERT(load >= 0 && load <= 1000); + ASSERT(load >= 0 && load <= kMaxLoad); } return oldLoad; @@ -1369,6 +1373,33 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) } +static inline void +update_cpu_performance(Thread* thread, int32 thisCore) +{ + int load = max_c(thread->scheduler_data->load, + sCoreEntries[thisCore].fLoad); + load = min_c(max_c(load, 0), kMaxLoad); + + if (load < kTargetLoad) { + int delta = kTargetLoad - load; + + delta *= kTargetLoad; + delta /= kCPUPerformanceScaleMax; + + decrease_cpu_performance(delta); + } else { + bool allowBoost = sSchedulerMode != SCHEDULER_MODE_POWER_SAVING; + allowBoost = allowBoost || thread->scheduler_data->priority_penalty > 0; + + int delta = load - kTargetLoad; + delta *= kMaxLoad - kTargetLoad; + delta /= kCPUPerformanceScaleMax; + + increase_cpu_performance(delta, allowBoost); + } +} + + static void _scheduler_reschedule(void) { @@ -1477,6 +1508,8 @@ _scheduler_reschedule(void) bigtime_t quantum = compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); + + update_cpu_performance(nextThread, thisCore); } else nextThread->scheduler_data->quantum_start = system_time(); From 9c0ff0eed12150e4b27b266af581b7d4758019a3 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 30 Oct 2013 00:55:03 +0100 Subject: [PATCH 085/273] kernel: Add cpufreq module for Intel P-states Since Sandy Bridge managing P-states on Intel processors is much easier and more powerful than when using previous versions of EIST. --- build/jam/packages/Haiku | 1 + build/jam/packages/HaikuBootstrap | 1 + headers/private/kernel/arch/x86/arch_cpu.h | 6 + src/add-ons/kernel/Jamfile | 1 + src/add-ons/kernel/power/Jamfile | 4 + src/add-ons/kernel/power/cpufreq/Jamfile | 4 + .../power/cpufreq/intel_pstates/Jamfile | 8 + .../cpufreq/intel_pstates/intel_pstates.cpp | 234 ++++++++++++++++++ 8 files changed, 259 insertions(+) create mode 100644 src/add-ons/kernel/power/Jamfile create mode 100644 src/add-ons/kernel/power/cpufreq/Jamfile create mode 100644 src/add-ons/kernel/power/cpufreq/intel_pstates/Jamfile create mode 100644 src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp diff --git a/build/jam/packages/Haiku b/build/jam/packages/Haiku index fa75457e8e..2ca840a6ff 100644 --- a/build/jam/packages/Haiku +++ b/build/jam/packages/Haiku @@ -52,6 +52,7 @@ AddFilesToPackage add-ons kernel generic AddFilesToPackage add-ons kernel partitioning_systems : amiga_rdb apple efi_gpt intel session ; AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ; +AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ; if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 { AddFilesToPackage add-ons kernel cpu : generic_x86 ; diff --git a/build/jam/packages/HaikuBootstrap b/build/jam/packages/HaikuBootstrap index 491ebaeeb4..f4a419d7d7 100644 --- a/build/jam/packages/HaikuBootstrap +++ b/build/jam/packages/HaikuBootstrap @@ -51,6 +51,7 @@ AddFilesToPackage add-ons kernel generic AddFilesToPackage add-ons kernel partitioning_systems : amiga_rdb apple efi_gpt intel session ; AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ; +AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ; if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 { AddFilesToPackage add-ons kernel cpu : generic_x86 ; diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index c51ee6f2fd..33cf17e4ae 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -31,10 +31,16 @@ #define IA32_MSR_TSC 0x10 #define IA32_MSR_APIC_BASE 0x1b +#define IA32_MSR_PLATFORM_INFO 0xce +#define IA32_MSR_MPERF 0xe7 +#define IA32_MSR_APERF 0xe8 #define IA32_MSR_MTRR_CAPABILITIES 0xfe #define IA32_MSR_SYSENTER_CS 0x174 #define IA32_MSR_SYSENTER_ESP 0x175 #define IA32_MSR_SYSENTER_EIP 0x176 +#define IA32_MSR_PERF_STATUS 0x198 +#define IA32_MSR_PERF_CTL 0x199 +#define IA32_MSR_TURBO_RATIO_LIMIT 0x1ad #define IA32_MSR_ENERGY_PERF_BIAS 0x1b0 #define IA32_MSR_MTRR_DEFAULT_TYPE 0x2ff #define IA32_MSR_MTRR_PHYSICAL_BASE_0 0x200 diff --git a/src/add-ons/kernel/Jamfile b/src/add-ons/kernel/Jamfile index 5a5b275cb3..a58d6382b9 100644 --- a/src/add-ons/kernel/Jamfile +++ b/src/add-ons/kernel/Jamfile @@ -14,3 +14,4 @@ SubInclude HAIKU_TOP src add-ons kernel media ; SubInclude HAIKU_TOP src add-ons kernel partitioning_systems ; SubInclude HAIKU_TOP src add-ons kernel generic ; SubInclude HAIKU_TOP src add-ons kernel bluetooth ; +SubInclude HAIKU_TOP src add-ons kernel power ; diff --git a/src/add-ons/kernel/power/Jamfile b/src/add-ons/kernel/power/Jamfile new file mode 100644 index 0000000000..84032da183 --- /dev/null +++ b/src/add-ons/kernel/power/Jamfile @@ -0,0 +1,4 @@ +SubDir HAIKU_TOP src add-ons kernel power ; + +SubInclude HAIKU_TOP src add-ons kernel power cpufreq ; + diff --git a/src/add-ons/kernel/power/cpufreq/Jamfile b/src/add-ons/kernel/power/cpufreq/Jamfile new file mode 100644 index 0000000000..163027559b --- /dev/null +++ b/src/add-ons/kernel/power/cpufreq/Jamfile @@ -0,0 +1,4 @@ +SubDir HAIKU_TOP src add-ons kernel power cpufreq ; + +SubInclude HAIKU_TOP src add-ons kernel power cpufreq intel_pstates ; + diff --git a/src/add-ons/kernel/power/cpufreq/intel_pstates/Jamfile b/src/add-ons/kernel/power/cpufreq/intel_pstates/Jamfile new file mode 100644 index 0000000000..22b8211d94 --- /dev/null +++ b/src/add-ons/kernel/power/cpufreq/intel_pstates/Jamfile @@ -0,0 +1,8 @@ +SubDir HAIKU_TOP src add-ons kernel power cpufreq intel_pstates ; + +UsePrivateKernelHeaders ; + +KernelAddon intel_pstates : + intel_pstates.cpp + ; + diff --git a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp new file mode 100644 index 0000000000..7af294bf9c --- /dev/null +++ b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp @@ -0,0 +1,234 @@ +/* + * Copyright 2013, Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Paweł Dziepak, + */ + + +#include +#include + +#include +#include +#include +#include + + +#define INTEL_PSTATES_MODULE_NAME CPUFREQ_MODULES_PREFIX "/intel_pstates/v1" + + +const int kMinimalInterval = 50000; + +static uint16 sMinPState; +static uint16 sMaxPState; +static uint16 sBoostPState; + + +struct CPUEntry { + CPUEntry(); + + uint16 fCurrentPState; + + bigtime_t fLastUpdate; +}; +static CPUEntry* sCPUEntries; + + +CPUEntry::CPUEntry() + : + fCurrentPState(sMinPState - 1), + fLastUpdate(0) +{ +} + + +static int +measure_pstate(CPUEntry* entry) +{ + InterruptsLocker locker; + + uint64 mperf = x86_read_msr(IA32_MSR_MPERF); + uint64 aperf = x86_read_msr(IA32_MSR_APERF); + + x86_write_msr(IA32_MSR_MPERF, 0); + x86_write_msr(IA32_MSR_APERF, 0); + + locker.Unlock(); + + if (mperf == 0) + return sMinPState; + + int oldPState = sMaxPState * aperf / mperf; + oldPState = min_c(max_c(oldPState, sMinPState), sBoostPState); + + return oldPState; +} + + +static inline void +set_pstate(uint16 pstate) +{ + CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()]; + pstate = min_c(max_c(sMinPState, pstate), sBoostPState); + + if (entry->fCurrentPState != pstate) { + entry->fLastUpdate = system_time(); + entry->fCurrentPState = pstate; + + x86_write_msr(IA32_MSR_PERF_CTL, pstate << 8); + } +} + + +static status_t +increase_performance(int delta, bool allowBoost) +{ + CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()]; + + if (system_time() - entry->fLastUpdate < kMinimalInterval) + return B_OK; + + int pState = measure_pstate(entry); + pState += (sBoostPState - pState) * delta / kCPUPerformanceScaleMax; + + if (!allowBoost) + pState = min_c(pState, sMaxPState); + + set_pstate(pState); + return B_OK; +} + + +static status_t +decrease_performance(int delta) +{ + CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()]; + + if (system_time() - entry->fLastUpdate < kMinimalInterval) + return B_OK; + + int pState = measure_pstate(entry); + pState -= (pState - sMinPState) * delta / kCPUPerformanceScaleMax; + + set_pstate(pState); + return B_OK; +} + + +static bool +is_cpu_model_supported(cpu_ent* cpu) +{ + uint8 model = cpu->arch.model + (cpu->arch.extended_model << 4); + + if (cpu->arch.vendor != VENDOR_INTEL) + return false; + + if (cpu->arch.family != 6) + return false; + + const uint8 kSupportedFamily6Models[] = { + 0x2a, 0x2d, 0x2e, 0x3a, 0x3c, 0x3e, 0x3f, 0x45, 0x46, + }; + const int kSupportedFamily6ModelsCount + = sizeof(kSupportedFamily6Models) / sizeof(uint8); + + int i; + for (i = 0; i < kSupportedFamily6ModelsCount; i++) { + if (model == kSupportedFamily6Models[i]) + break; + } + + return i != kSupportedFamily6ModelsCount; +} + + +static void +set_normal_pstate(void* /* dummy */, int cpu) +{ + measure_pstate(&sCPUEntries[cpu]); + set_pstate(sMaxPState); +} + + +static status_t +init_pstates() +{ + if (!x86_check_feature(IA32_FEATURE_MSR, FEATURE_COMMON)) + return B_ERROR; + + if (!x86_check_feature(IA32_FEATURE_APERFMPERF, FEATURE_6_ECX)) + return B_ERROR; + + int32 cpuCount = smp_get_num_cpus(); + for (int32 i = 0; i < cpuCount; i++) { + if (!is_cpu_model_supported(&gCPU[i])) + return B_ERROR; + } + + sMinPState = (x86_read_msr(IA32_MSR_PLATFORM_INFO) >> 40) & 0xff; + sMaxPState = (x86_read_msr(IA32_MSR_PLATFORM_INFO) >> 8) & 0xff; + sBoostPState + = max_c(x86_read_msr(IA32_MSR_TURBO_RATIO_LIMIT) & 0xff, sMaxPState); + + dprintf("using Intel P-States: min %" B_PRIu16 ", max %" B_PRIu16 + ", boost %" B_PRIu16 "\n", sMinPState, sMaxPState, sBoostPState); + + if (sMaxPState <= sMinPState || sMaxPState == 0) { + dprintf("unexpected or invalid Intel P-States limits, aborting\n"); + return B_ERROR; + } + + sCPUEntries = new(std::nothrow) CPUEntry[cpuCount]; + if (sCPUEntries == NULL) + return B_NO_MEMORY; + + call_all_cpus_sync(set_normal_pstate, NULL); + return B_OK; +} + + +static status_t +uninit_pstates() +{ + call_all_cpus_sync(set_normal_pstate, NULL); + delete[] sCPUEntries; + + return B_OK; +} + + +static status_t +std_ops(int32 op, ...) +{ + switch (op) { + case B_MODULE_INIT: + return init_pstates(); + + case B_MODULE_UNINIT: + uninit_pstates(); + return B_OK; + } + + return B_ERROR; +} + + +static cpufreq_module_info sIntelPStates = { + { + INTEL_PSTATES_MODULE_NAME, + 0, + std_ops, + }, + + increase_performance, + decrease_performance, +}; + + +module_info* modules[] = { + (module_info*)&sIntelPStates, + NULL +}; + From d54a9e0a4194f6843406bc67f68045f59f9a190d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 30 Oct 2013 02:57:45 +0100 Subject: [PATCH 086/273] kernel: Do not use gSchedulerLock when accesing UID and GID Reads and writes to uid_t and gid_t are atomic anyway. The only real problem that may happen here is inconsistent state of triples effective_{u, g}id, saved_set_{u, g}id, real_{u, g}id, but team locks protect us against that. --- src/system/kernel/signal.cpp | 15 ++++----------- src/system/kernel/usergroup.cpp | 26 +++----------------------- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/src/system/kernel/signal.cpp b/src/system/kernel/signal.cpp index a28533f645..1d61b15202 100644 --- a/src/system/kernel/signal.cpp +++ b/src/system/kernel/signal.cpp @@ -315,7 +315,6 @@ Signal::SetTo(uint32 number) fErrorCode = 0; fSendingProcess = team->id; fSendingUser = team->effective_uid; - // assuming scheduler lock is being held fStatus = 0; fPollBand = 0; fAddress = NULL; @@ -1322,19 +1321,13 @@ has_signals_pending(Thread* thread) /*! Checks whether the current user has permission to send a signal to the given target team. - The caller must hold the scheduler lock or \a team's lock. - \param team The target team. - \param schedulerLocked \c true, if the caller holds the scheduler lock, - \c false otherwise. */ static bool -has_permission_to_signal(Team* team, bool schedulerLocked) +has_permission_to_signal(Signal* signal, Team* team) { // get the current user - uid_t currentUser = schedulerLocked - ? thread_get_current_thread()->team->effective_uid - : geteuid(); + uid_t currentUser = signal->SendingUser(); // root is omnipotent -- in the other cases the current user must match the // target team's @@ -1372,7 +1365,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, BReference signalReference(signal, true); if ((flags & B_CHECK_PERMISSION) != 0) { - if (!has_permission_to_signal(thread->team, true)) + if (!has_permission_to_signal(signal, thread->team)) return EPERM; } @@ -1566,7 +1559,7 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, BReference signalReference(signal, true); if ((flags & B_CHECK_PERMISSION) != 0) { - if (!has_permission_to_signal(team, true)) + if (!has_permission_to_signal(signal, team)) return EPERM; } diff --git a/src/system/kernel/usergroup.cpp b/src/system/kernel/usergroup.cpp index 5a54ea07c9..9be52f5a0c 100644 --- a/src/system/kernel/usergroup.cpp +++ b/src/system/kernel/usergroup.cpp @@ -54,7 +54,6 @@ common_setregid(gid_t rgid, gid_t egid, bool setAllIfPrivileged, bool kernel) // setgid() semantics: If privileged set both, real, effective and // saved set-gid, otherwise set the effective gid. if (privileged) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); team->saved_set_gid = rgid; team->real_gid = rgid; team->effective_gid = rgid; @@ -91,7 +90,6 @@ common_setregid(gid_t rgid, gid_t egid, bool setAllIfPrivileged, bool kernel) } // Getting here means all checks were successful -- set the gids. - InterruptsSpinLocker schedulerLocker(gSchedulerLock); team->real_gid = rgid; team->effective_gid = egid; team->saved_set_gid = ssgid; @@ -119,7 +117,6 @@ common_setreuid(uid_t ruid, uid_t euid, bool setAllIfPrivileged, bool kernel) // setuid() semantics: If privileged set both, real, effective and // saved set-uid, otherwise set the effective uid. if (privileged) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); team->saved_set_uid = ruid; team->real_uid = ruid; team->effective_uid = ruid; @@ -156,7 +153,6 @@ common_setreuid(uid_t ruid, uid_t euid, bool setAllIfPrivileged, bool kernel) } // Getting here means all checks were successful -- set the uids. - InterruptsSpinLocker schedulerLocker(gSchedulerLock); team->real_uid = ruid; team->effective_uid = euid; team->saved_set_uid = ssuid; @@ -253,8 +249,6 @@ common_setgroups(int groupCount, const gid_t* groupList, bool kernel) void inherit_parent_user_and_group(Team* team, Team* parent) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - team->saved_set_uid = parent->saved_set_uid; team->real_uid = parent->real_uid; team->effective_uid = parent->effective_uid; @@ -262,8 +256,6 @@ inherit_parent_user_and_group(Team* team, Team* parent) team->real_gid = parent->real_gid; team->effective_gid = parent->effective_gid; - schedulerLocker.Unlock(); - malloc_referenced_acquire(parent->supplementary_groups); team->supplementary_groups = parent->supplementary_groups; team->supplementary_group_count = parent->supplementary_group_count; @@ -279,7 +271,6 @@ update_set_id_user_and_group(Team* team, const char* file) return status; TeamLocker teamLocker(team); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); if ((st.st_mode & S_ISUID) != 0) { team->saved_set_uid = st.st_uid; @@ -300,8 +291,6 @@ _kern_getgid(bool effective) { Team* team = thread_get_current_thread()->team; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - return effective ? team->effective_gid : team->real_gid; } @@ -311,8 +300,6 @@ _kern_getuid(bool effective) { Team* team = thread_get_current_thread()->team; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - return effective ? team->effective_uid : team->real_uid; } @@ -353,8 +340,6 @@ _user_getgid(bool effective) { Team* team = thread_get_current_thread()->team; - TeamLocker teamLocker(team); - return effective ? team->effective_gid : team->real_gid; } @@ -364,8 +349,6 @@ _user_getuid(bool effective) { Team* team = thread_get_current_thread()->team; - TeamLocker teamLocker(team); - return effective ? team->effective_uid : team->real_uid; } @@ -395,12 +378,9 @@ ssize_t _user_setgroups(int groupCount, const gid_t* groupList) { // check privilege - { - Team* team = thread_get_current_thread()->team; - TeamLocker teamLocker(team); - if (!is_privileged(team)) - return EPERM; - } + Team* team = thread_get_current_thread()->team; + if (!is_privileged(team)) + return EPERM; return common_setgroups(groupCount, groupList, false); } From d70728f54da5b22311b6f271a60fe2067ae4a47b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 30 Oct 2013 03:26:13 +0100 Subject: [PATCH 087/273] kernel/lock: Do not use *_locked() functions when not needed --- src/system/kernel/locks/lock.cpp | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/system/kernel/locks/lock.cpp b/src/system/kernel/locks/lock.cpp index 3b1d069c44..7f4094f953 100644 --- a/src/system/kernel/locks/lock.cpp +++ b/src/system/kernel/locks/lock.cpp @@ -168,9 +168,7 @@ rw_lock_wait(rw_lock* lock, bool writer, InterruptsSpinLocker& locker) thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_RW_LOCK, lock); locker.Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - status_t result = thread_block_locked(thread_get_current_thread()); - schedulerLocker.Unlock(); + status_t result = thread_block(); locker.Lock(); return result; @@ -386,10 +384,7 @@ _rw_lock_read_lock_with_timeout(rw_lock* lock, uint32 timeoutFlags, thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_RW_LOCK, lock); locker.Unlock(); - InterruptsSpinLocker schedulerLock(gSchedulerLock); - status_t error = thread_block_with_timeout_locked(timeoutFlags, timeout); - schedulerLock.Unlock(); - + status_t error = thread_block_with_timeout(timeoutFlags, timeout); if (error == B_OK || waiter.thread == NULL) { // We were unblocked successfully -- potentially our unblocker overtook // us after we already failed. In either case, we've got the lock, now. @@ -752,10 +747,7 @@ _mutex_lock(mutex* lock, void* _locker) thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_MUTEX, lock); locker->Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - status_t error = thread_block_locked(waiter.thread); - schedulerLocker.Unlock(); - + status_t error = thread_block(); #if KDEBUG if (error == B_OK) atomic_set(&lock->holder, waiter.thread->id); @@ -875,16 +867,15 @@ _mutex_lock_with_timeout(mutex* lock, uint32 timeoutFlags, bigtime_t timeout) thread_prepare_to_block(waiter.thread, 0, THREAD_BLOCK_TYPE_MUTEX, lock); locker.Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - status_t error = thread_block_with_timeout_locked(timeoutFlags, timeout); - schedulerLocker.Unlock(); + status_t error = thread_block_with_timeout(timeoutFlags, timeout); - locker.Lock(); if (error == B_OK) { #if KDEBUG lock->holder = waiter.thread->id; #endif } else { + locker.Lock(); + // If the timeout occurred, we must remove our waiter structure from // the queue. mutex_waiter* previousWaiter = NULL; From c8dd9f7780c426e592a3ccb231e6bfab51f15eb9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 30 Oct 2013 03:58:36 +0100 Subject: [PATCH 088/273] kernel: Add thread_unblock() and use it where possible --- headers/private/kernel/thread.h | 1 + src/system/kernel/debug/system_profiler.cpp | 3 +-- src/system/kernel/fs/Vnode.cpp | 3 +-- src/system/kernel/fs/fifo.cpp | 3 +-- src/system/kernel/locks/lock.cpp | 18 +++++------------- src/system/kernel/posix/xsi_message_queue.cpp | 12 ++++-------- src/system/kernel/posix/xsi_semaphore.cpp | 11 ++++------- src/system/kernel/sem.cpp | 4 +--- src/system/kernel/thread.cpp | 13 +++++++++++++ src/system/kernel/vm/VMCache.cpp | 3 +-- src/system/kernel/vm/vm_page.cpp | 3 +-- 11 files changed, 33 insertions(+), 41 deletions(-) diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 469e339290..ecb6245dbd 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -132,6 +132,7 @@ status_t thread_block(); status_t thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout); status_t thread_block_with_timeout_locked(uint32 timeoutFlags, bigtime_t timeout); +void thread_unblock(Thread* thread, status_t status); // used in syscalls.c status_t _user_set_thread_priority(thread_id thread, int32 newPriority); diff --git a/src/system/kernel/debug/system_profiler.cpp b/src/system/kernel/debug/system_profiler.cpp index c2d75492cc..d0d3e22016 100644 --- a/src/system/kernel/debug/system_profiler.cpp +++ b/src/system/kernel/debug/system_profiler.cpp @@ -298,8 +298,7 @@ SystemProfiler::~SystemProfiler() // inactive. InterruptsSpinLocker locker(fLock); if (fWaitingProfilerThread != NULL) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(fWaitingProfilerThread, B_OK); + thread_unblock(fWaitingProfilerThread, B_OK); fWaitingProfilerThread = NULL; } fProfilingActive = false; diff --git a/src/system/kernel/fs/Vnode.cpp b/src/system/kernel/fs/Vnode.cpp index 890e29bf9f..cc71aae551 100644 --- a/src/system/kernel/fs/Vnode.cpp +++ b/src/system/kernel/fs/Vnode.cpp @@ -88,6 +88,5 @@ vnode::_WakeUpLocker() atomic_and(&fFlags, ~kFlagsWaitingLocker); // and wake it up - InterruptsSpinLocker threadLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_OK); + thread_unblock(waiter->thread, B_OK); } diff --git a/src/system/kernel/fs/fifo.cpp b/src/system/kernel/fs/fifo.cpp index 9c771ff816..fe2ffb6959 100644 --- a/src/system/kernel/fs/fifo.cpp +++ b/src/system/kernel/fs/fifo.cpp @@ -100,8 +100,7 @@ public: TRACE("ReadRequest %p::Notify(), fNotified %d\n", this, fNotified); if (!fNotified) { - SpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(fThread, status); + thread_unblock(fThread, status); fNotified = true; } } diff --git a/src/system/kernel/locks/lock.cpp b/src/system/kernel/locks/lock.cpp index 7f4094f953..6b3ac5ff70 100644 --- a/src/system/kernel/locks/lock.cpp +++ b/src/system/kernel/locks/lock.cpp @@ -197,9 +197,7 @@ rw_lock_unblock(rw_lock* lock) lock->holder = waiter->thread->id; // unblock thread - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_OK); - schedulerLocker.Unlock(); + thread_unblock(waiter->thread, B_OK); waiter->thread = NULL; return RW_LOCK_WRITER_COUNT_BASE; @@ -216,9 +214,7 @@ rw_lock_unblock(rw_lock* lock) readerCount++; // unblock thread - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_OK); - schedulerLocker.Unlock(); + thread_unblock(waiter->thread, B_OK); waiter->thread = NULL; } while ((waiter = lock->waiters) != NULL && !waiter->writer); @@ -293,8 +289,7 @@ rw_lock_destroy(rw_lock* lock) lock->waiters = waiter->next; // unblock thread - InterruptsSpinLocker _(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_ERROR); + thread_unblock(waiter->thread, B_ERROR); } lock->name = NULL; @@ -637,8 +632,7 @@ mutex_destroy(mutex* lock) lock->waiters = waiter->next; // unblock thread - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_ERROR); + thread_unblock(waiter->thread, B_ERROR); } lock->name = NULL; @@ -783,9 +777,7 @@ _mutex_unlock(mutex* lock) lock->waiters->last = waiter->last; // unblock thread - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_OK); - schedulerLocker.Unlock(); + thread_unblock(waiter->thread, B_OK); #if KDEBUG // Already set the holder to the unblocked thread. Besides that this diff --git a/src/system/kernel/posix/xsi_message_queue.cpp b/src/system/kernel/posix/xsi_message_queue.cpp index 278c241f67..f0141b1e93 100644 --- a/src/system/kernel/posix/xsi_message_queue.cpp +++ b/src/system/kernel/posix/xsi_message_queue.cpp @@ -246,8 +246,6 @@ public: void WakeUpThread(bool waitForMessage) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - if (waitForMessage) { // Wake up all waiting thread for a message // TODO: this can cause starvation for any @@ -255,14 +253,14 @@ public: while (queued_thread *entry = fWaitingToReceive.RemoveHead()) { entry->queued = false; fThreadsWaitingToReceive--; - thread_unblock_locked(entry->thread, 0); + thread_unblock(entry->thread, 0); } } else { // Wake up only one thread waiting to send if (queued_thread *entry = fWaitingToSend.RemoveHead()) { entry->queued = false; fThreadsWaitingToSend--; - thread_unblock_locked(entry->thread, 0); + thread_unblock(entry->thread, 0); } } } @@ -400,15 +398,13 @@ XsiMessageQueue::~XsiMessageQueue() // Wake up any threads still waiting if (fThreadsWaitingToSend || fThreadsWaitingToReceive) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - while (queued_thread *entry = fWaitingToReceive.RemoveHead()) { entry->queued = false; - thread_unblock_locked(entry->thread, EIDRM); + thread_unblock(entry->thread, EIDRM); } while (queued_thread *entry = fWaitingToSend.RemoveHead()) { entry->queued = false; - thread_unblock_locked(entry->thread, EIDRM); + thread_unblock(entry->thread, EIDRM); } } diff --git a/src/system/kernel/posix/xsi_semaphore.cpp b/src/system/kernel/posix/xsi_semaphore.cpp index 212ee1b592..4c76c799cf 100644 --- a/src/system/kernel/posix/xsi_semaphore.cpp +++ b/src/system/kernel/posix/xsi_semaphore.cpp @@ -101,15 +101,13 @@ public: { // For some reason the semaphore is getting destroyed. // Wake up any remaing awaiting threads - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - while (queued_thread *entry = fWaitingToIncreaseQueue.RemoveHead()) { entry->queued = false; - thread_unblock_locked(entry->thread, EIDRM); + thread_unblock(entry->thread, EIDRM); } while (queued_thread *entry = fWaitingToBeZeroQueue.RemoveHead()) { entry->queued = false; - thread_unblock_locked(entry->thread, EIDRM); + thread_unblock(entry->thread, EIDRM); } // No need to remove any sem_undo request still // hanging. When the process exit and doesn't found @@ -218,20 +216,19 @@ public: void WakeUpThread(bool waitingForZero) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); if (waitingForZero) { // Wake up all threads waiting on zero while (queued_thread *entry = fWaitingToBeZeroQueue.RemoveHead()) { entry->queued = false; fThreadsWaitingToBeZero--; - thread_unblock_locked(entry->thread, 0); + thread_unblock(entry->thread, 0); } } else { // Wake up all threads even though they might go back to sleep while (queued_thread *entry = fWaitingToIncreaseQueue.RemoveHead()) { entry->queued = false; fThreadsWaitingToIncrease--; - thread_unblock_locked(entry->thread, 0); + thread_unblock(entry->thread, 0); } } } diff --git a/src/system/kernel/sem.cpp b/src/system/kernel/sem.cpp index 0b761c2724..7f01069291 100644 --- a/src/system/kernel/sem.cpp +++ b/src/system/kernel/sem.cpp @@ -330,12 +330,10 @@ uninit_sem_locked(struct sem_entry& sem, char** _name) sem.u.used.select_infos = NULL; // free any threads waiting for this semaphore - SpinLocker schedulerLocker(gSchedulerLock); while (queued_thread* entry = sem.queue.RemoveHead()) { entry->queued = false; - thread_unblock_locked(entry->thread, B_BAD_SEM_ID); + thread_unblock(entry->thread, B_BAD_SEM_ID); } - schedulerLocker.Unlock(); int32 id = sem.id; sem.id = -1; diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 210492d15e..20bf8f79ad 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2889,6 +2889,19 @@ thread_block_with_timeout_locked(uint32 timeoutFlags, bigtime_t timeout) } +/*! Unblocks a thread. + + Acquires the scheduler lock and calls thread_unblock_locked(). + See there for more information. +*/ +void +thread_unblock(Thread* thread, status_t status) +{ + InterruptsSpinLocker _(gSchedulerLock); + thread_unblock_locked(thread, status); +} + + /*! Unblocks a userland-blocked thread. The caller must not hold any locks. */ diff --git a/src/system/kernel/vm/VMCache.cpp b/src/system/kernel/vm/VMCache.cpp index d9efe304bd..a1d16677bf 100644 --- a/src/system/kernel/vm/VMCache.cpp +++ b/src/system/kernel/vm/VMCache.cpp @@ -1383,8 +1383,7 @@ VMCache::_NotifyPageEvents(vm_page* page, uint32 events) if (waiter->page == page && (waiter->events & events) != 0) { // remove from list and unblock *it = waiter->next; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_OK); + thread_unblock(waiter->thread, B_OK); } else it = &waiter->next; } diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 35d1a8bfee..dd9b9b3d48 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -1447,8 +1447,7 @@ wake_up_page_reservation_waiters() sPageReservationWaiters.Remove(waiter); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_unblock_locked(waiter->thread, B_OK); + thread_unblock(waiter->thread, B_OK); } } From d8fcc8a82519cef977c689cee497316be6f1531f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 31 Oct 2013 01:49:43 +0100 Subject: [PATCH 089/273] kernel: Remove B_TIMER_ACQUIRE_SCHEDULER_LOCK flag The flag main purpose is to avoid race conditions between event handler and cancel_timer(). However, cancel_timer() is safe even without using gSchedulerLock. If the event is scheduled to happen on other CPU than the CPU that invokes cancel_timer() then cancel_timer() either disables the event before its handler starts executing or waits until the event handler is done. If the event is scheduled on the same CPU that calls cancel_timer() then, since cancel_timer() disables interrupts, the event is either executed before cancel_timer() or when the timer interrupt handler starts running the event is already disabled. --- headers/private/kernel/timer.h | 7 +------ src/system/kernel/UserTimer.cpp | 16 ++++----------- src/system/kernel/scheduler/scheduler.cpp | 2 +- src/system/kernel/thread.cpp | 19 +++--------------- src/system/kernel/timer.cpp | 24 +++-------------------- 5 files changed, 12 insertions(+), 56 deletions(-) diff --git a/headers/private/kernel/timer.h b/headers/private/kernel/timer.h index 2789b56493..a1c91af824 100644 --- a/headers/private/kernel/timer.h +++ b/headers/private/kernel/timer.h @@ -23,13 +23,8 @@ struct kernel_args; #define B_TIMER_USE_TIMER_STRUCT_TIMES 0x4000 // For add_timer(): Use the timer::schedule_time (absolute time) and // timer::period values instead of the period parameter. -#define B_TIMER_ACQUIRE_SCHEDULER_LOCK 0x8000 - // The timer hook is invoked with the scheduler lock held. When invoking - // cancel_timer() with the scheduler lock held, too, this helps to avoid - // race conditions. #define B_TIMER_FLAGS \ - (B_TIMER_USE_TIMER_STRUCT_TIMES | B_TIMER_ACQUIRE_SCHEDULER_LOCK \ - | B_TIMER_REAL_TIME_BASE) + (B_TIMER_USE_TIMER_STRUCT_TIMES | B_TIMER_REAL_TIME_BASE) /* Timer info structure */ struct timer_info { diff --git a/src/system/kernel/UserTimer.cpp b/src/system/kernel/UserTimer.cpp index 8c0cabec15..70e6082a2b 100644 --- a/src/system/kernel/UserTimer.cpp +++ b/src/system/kernel/UserTimer.cpp @@ -191,6 +191,7 @@ UserTimer::Cancel() /*static*/ int32 UserTimer::HandleTimerHook(struct timer* timer) { + InterruptsSpinLocker _(gSchedulerLock); ((UserTimer*)timer->user_data)->HandleTimer(); return B_HANDLED_INTERRUPT; } @@ -352,9 +353,7 @@ SystemTimeUserTimer::ScheduleKernelTimer(bigtime_t now, CheckPeriodicOverrun(now); uint32 timerFlags = B_ONE_SHOT_ABSOLUTE_TIMER - | B_TIMER_USE_TIMER_STRUCT_TIMES | B_TIMER_ACQUIRE_SCHEDULER_LOCK; - // We use B_TIMER_ACQUIRE_SCHEDULER_LOCK to avoid race conditions - // between setting/canceling the timer and the event handler. + | B_TIMER_USE_TIMER_STRUCT_TIMES; fTimer.schedule_time = std::max(fNextTime, (bigtime_t)0); fTimer.period = 0; @@ -692,10 +691,7 @@ TeamTimeUserTimer::_Update(bool unscheduling) // rounding errors. add_timer(&fTimer, &HandleTimerHook, fTimer.schedule_time, - B_ONE_SHOT_ABSOLUTE_TIMER | B_TIMER_USE_TIMER_STRUCT_TIMES - | B_TIMER_ACQUIRE_SCHEDULER_LOCK); - // We use B_TIMER_ACQUIRE_SCHEDULER_LOCK to avoid race conditions - // between setting/canceling the timer and the event handler. + B_ONE_SHOT_ABSOLUTE_TIMER | B_TIMER_USE_TIMER_STRUCT_TIMES); // We use B_TIMER_USE_TIMER_STRUCT_TIMES, so period remains 0, which // our base class expects. @@ -989,11 +985,7 @@ ThreadTimeUserTimer::Start() fTimer.schedule_time = 0; fTimer.period = 0; - uint32 flags = B_ONE_SHOT_ABSOLUTE_TIMER - | B_TIMER_USE_TIMER_STRUCT_TIMES | B_TIMER_ACQUIRE_SCHEDULER_LOCK; - // We use B_TIMER_ACQUIRE_SCHEDULER_LOCK to avoid race conditions - // between setting/canceling the timer and the event handler. - + uint32 flags = B_ONE_SHOT_ABSOLUTE_TIMER | B_TIMER_USE_TIMER_STRUCT_TIMES; add_timer(&fTimer, &HandleTimerHook, fTimer.schedule_time, flags); fScheduled = true; diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 7a90a8f930..87f564a5a3 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1507,7 +1507,7 @@ _scheduler_reschedule(void) if (!thread_is_idle_thread(nextThread)) { bigtime_t quantum = compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, - B_ONE_SHOT_RELATIVE_TIMER | B_TIMER_ACQUIRE_SCHEDULER_LOCK); + B_ONE_SHOT_RELATIVE_TIMER); update_cpu_performance(nextThread, thisCore); } else diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 20bf8f79ad..9fd94c9776 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1923,13 +1923,8 @@ thread_exit(void) } if (team != kernelTeam) { - // Cancel previously installed alarm timer, if any. Hold the scheduler - // lock to make sure that when cancel_timer() returns, the alarm timer - // hook will not be invoked anymore (since - // B_TIMER_ACQUIRE_SCHEDULER_LOCK is used). - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + // Cancel previously installed alarm timer, if any. cancel_timer(&thread->alarm); - schedulerLocker.Unlock(); // Delete all user timers associated with the thread. ThreadLocker threadLocker(thread); @@ -2782,12 +2777,8 @@ thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum) static status_t thread_block_timeout(timer* timer) { - // The timer has been installed with B_TIMER_ACQUIRE_SCHEDULER_LOCK, so - // we're holding the scheduler lock already. This makes things comfortably - // easy. - Thread* thread = (Thread*)timer->user_data; - thread_unblock_locked(thread, B_TIMED_OUT); + thread_unblock(thread, B_TIMED_OUT); return B_HANDLED_INTERRUPT; } @@ -2858,10 +2849,7 @@ thread_block_with_timeout_locked(uint32 timeoutFlags, bigtime_t timeout) && timeout != B_INFINITE_TIMEOUT; if (useTimer) { - // Timer flags: absolute/relative + "acquire thread lock". The latter - // avoids nasty race conditions and deadlock problems that could - // otherwise occur between our cancel_timer() and a concurrently - // executing thread_block_timeout(). + // Timer flags: absolute/relative. uint32 timerFlags; if ((timeoutFlags & B_RELATIVE_TIMEOUT) != 0) { timerFlags = B_ONE_SHOT_RELATIVE_TIMER; @@ -2870,7 +2858,6 @@ thread_block_with_timeout_locked(uint32 timeoutFlags, bigtime_t timeout) if ((timeoutFlags & B_TIMEOUT_REAL_TIME_BASE) != 0) timerFlags |= B_TIMER_REAL_TIME_BASE; } - timerFlags |= B_TIMER_ACQUIRE_SCHEDULER_LOCK; // install the timer thread->wait.unblock_timer.user_data = thread; diff --git a/src/system/kernel/timer.cpp b/src/system/kernel/timer.cpp index 3f626c9eeb..afea3a79d2 100644 --- a/src/system/kernel/timer.cpp +++ b/src/system/kernel/timer.cpp @@ -274,25 +274,8 @@ timer_interrupt() // call the callback // note: if the event is not periodic, it is ok // to delete the event structure inside the callback - if (event->hook) { - bool callHook = true; - - // we may need to acquire the scheduler lock - if ((mode & B_TIMER_ACQUIRE_SCHEDULER_LOCK) != 0) { - acquire_spinlock(&gSchedulerLock); - - // If the event has been cancelled in the meantime, we don't - // call the hook anymore. - if (cpuData.current_event == NULL) - callHook = false; - } - - if (callHook) - rc = event->hook(event); - - if ((mode & B_TIMER_ACQUIRE_SCHEDULER_LOCK) != 0) - release_spinlock(&gSchedulerLock); - } + if (event->hook) + rc = event->hook(event); cpuData.current_event_in_progress = 0; @@ -461,8 +444,7 @@ cancel_timer(timer* event) // lock to be held while calling the event hook, we'll have to wait // for the hook to complete. When called from the timer hook we don't // wait either, of course. - if ((event->flags & B_TIMER_ACQUIRE_SCHEDULER_LOCK) == 0 - && cpu != smp_get_current_cpu()) { + if (cpu != smp_get_current_cpu()) { spinLocker.Unlock(); while (cpuData.current_event_in_progress == 1) { From c2763aaffbd34bd92065b97c0f2770e418824772 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 31 Oct 2013 02:34:09 +0100 Subject: [PATCH 090/273] kernel: Add spinlock for undertaker data --- src/system/kernel/thread.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 9fd94c9776..e11fea41ed 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -141,6 +141,7 @@ public: static DoublyLinkedList sUndertakerEntries; +static spinlock sUndertakerLock = B_SPINLOCK_INITIALIZER; static ConditionVariable sUndertakerCondition; static ThreadNotificationService sNotificationService; @@ -1076,20 +1077,20 @@ undertaker(void* /*args*/) { while (true) { // wait for a thread to bury - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(sUndertakerLock); while (sUndertakerEntries.IsEmpty()) { ConditionVariableEntry conditionEntry; sUndertakerCondition.Add(&conditionEntry); - schedulerLocker.Unlock(); + locker.Unlock(); conditionEntry.Wait(); - schedulerLocker.Lock(); + locker.Lock(); } UndertakerEntry* _entry = sUndertakerEntries.RemoveHead(); - schedulerLocker.Unlock(); + locker.Unlock(); UndertakerEntry entry = *_entry; // we need a copy, since the original entry is on the thread's stack @@ -1102,7 +1103,7 @@ undertaker(void* /*args*/) Team* kernelTeam = team_get_kernel_team(); TeamLocker kernelTeamLocker(kernelTeam); thread->Lock(); - schedulerLocker.Lock(); + InterruptsSpinLocker schedulerLocker(gSchedulerLock); remove_thread_from_team(kernelTeam, thread); @@ -2230,11 +2231,13 @@ thread_exit(void) UndertakerEntry undertakerEntry(thread, teamID); disable_interrupts(); - schedulerLocker.Lock(); + SpinLocker undertakerLocker(sUndertakerLock); sUndertakerEntries.Add(&undertakerEntry); - sUndertakerCondition.NotifyOne(true); + sUndertakerCondition.NotifyOne(); + undertakerLocker.Unlock(); + schedulerLocker.Lock(); thread->next_state = THREAD_STATE_FREE_ON_RESCHED; scheduler_reschedule(); From 11cacd0c1345cd3296f12e07d18996a2c7572a64 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 4 Nov 2013 23:45:14 +0100 Subject: [PATCH 091/273] kernel: Remove thread_block_with_timeout_locked() --- headers/private/kernel/thread.h | 18 +++++++-------- src/system/kernel/condition_variable.cpp | 8 ++----- src/system/kernel/sem.cpp | 6 +---- src/system/kernel/signal.cpp | 10 ++++++-- src/system/kernel/thread.cpp | 29 +++++++----------------- 5 files changed, 27 insertions(+), 44 deletions(-) diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index ecb6245dbd..9cc12499c2 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -130,8 +130,6 @@ status_t deselect_thread(int32 object, struct select_info *info, bool kernel); status_t thread_block(); status_t thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout); -status_t thread_block_with_timeout_locked(uint32 timeoutFlags, - bigtime_t timeout); void thread_unblock(Thread* thread, status_t status); // used in syscalls.c @@ -230,22 +228,22 @@ thread_is_blocked(Thread* thread) If a client lock other than the scheduler lock is used, this function must be called with that lock being held. Afterwards that lock should be dropped and the function that actually blocks the thread shall be invoked - (thread_block[_locked]() or thread_block_with_timeout[_locked]()). In - between these two steps no functionality that uses the thread blocking API - for this thread shall be used. + (thread_block[_locked]() or thread_block_with_timeout()). In between these + two steps no functionality that uses the thread blocking API for this thread + shall be used. When the caller determines that the condition for unblocking the thread occurred, it calls thread_unblock_locked() to unblock the thread. At that time one of locks that are held when calling thread_prepare_to_block() must be held. Usually that would be the client lock. In two cases it generally isn't, however, since the unblocking code doesn't know about the client - lock: 1. When thread_block_with_timeout[_locked]() had been used and the - timeout occurs. 2. When thread_prepare_to_block() had been called with one - or both of the \c B_CAN_INTERRUPT or \c B_KILL_CAN_INTERRUPT flags specified - and someone calls thread_interrupt() that is supposed to wake up the thread. + lock: 1. When thread_block_with_timeout() had been used and the timeout + occurs. 2. When thread_prepare_to_block() had been called with one or both + of the \c B_CAN_INTERRUPT or \c B_KILL_CAN_INTERRUPT flags specified and + someone calls thread_interrupt() that is supposed to wake up the thread. In either of these two cases only the scheduler lock is held by the unblocking code. A timeout can only happen after - thread_block_with_timeout_locked() has been called, but an interruption is + thread_block_with_timeout() has been called, but an interruption is possible at any time. The client code must deal with those situations. Generally blocking and unblocking threads proceed in the following manner: diff --git a/src/system/kernel/condition_variable.cpp b/src/system/kernel/condition_variable.cpp index 47215606b1..7f6152ebce 100644 --- a/src/system/kernel/condition_variable.cpp +++ b/src/system/kernel/condition_variable.cpp @@ -134,15 +134,11 @@ ConditionVariableEntry::Wait(uint32 flags, bigtime_t timeout) conditionLocker.Unlock(); - SpinLocker schedulerLocker(gSchedulerLock); - status_t error; if ((flags & (B_RELATIVE_TIMEOUT | B_ABSOLUTE_TIMEOUT)) != 0) - error = thread_block_with_timeout_locked(flags, timeout); + error = thread_block_with_timeout(flags, timeout); else - error = thread_block_locked(thread_get_current_thread()); - - schedulerLocker.Unlock(); + error = thread_block(); conditionLocker.Lock(); diff --git a/src/system/kernel/sem.cpp b/src/system/kernel/sem.cpp index 7f01069291..7980ed29e2 100644 --- a/src/system/kernel/sem.cpp +++ b/src/system/kernel/sem.cpp @@ -852,13 +852,9 @@ switch_sem_etc(sem_id semToBeReleased, sem_id id, int32 count, semToBeReleased = -1; } - schedulerLocker.Lock(); - status_t acquireStatus = timeout == B_INFINITE_TIMEOUT - ? thread_block_locked(thread) - : thread_block_with_timeout_locked(flags, timeout); + ? thread_block() : thread_block_with_timeout(flags, timeout); - schedulerLocker.Unlock(); GRAB_SEM_LOCK(sSems[slot]); // If we're still queued, this means the acquiration failed, and we diff --git a/src/system/kernel/signal.cpp b/src/system/kernel/signal.cpp index 1d61b15202..751d763f6e 100644 --- a/src/system/kernel/signal.cpp +++ b/src/system/kernel/signal.cpp @@ -2041,16 +2041,22 @@ sigwait_internal(const sigset_t* set, siginfo_t* info, uint32 flags, thread_prepare_to_block(thread, flags, THREAD_BLOCK_TYPE_SIGNAL, NULL); + schedulerLocker.Unlock(); + if ((flags & B_ABSOLUTE_TIMEOUT) != 0) { - error = thread_block_with_timeout_locked(flags, timeout); + error = thread_block_with_timeout(flags, timeout); if (error == B_WOULD_BLOCK || error == B_TIMED_OUT) { error = B_WOULD_BLOCK; // POSIX requires EAGAIN (B_WOULD_BLOCK) on timeout timedOut = true; + + schedulerLocker.Lock(); break; } } else - thread_block_locked(thread); + thread_block(); + + schedulerLocker.Lock(); } // restore the original block mask diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index e11fea41ed..d36b212a09 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1364,11 +1364,9 @@ common_snooze_etc(bigtime_t timeout, clockid_t clockID, uint32 flags, Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_prepare_to_block(thread, flags, THREAD_BLOCK_TYPE_SNOOZE, NULL); - status_t status = thread_block_with_timeout_locked(flags, timeout); + status_t status = thread_block_with_timeout(flags, timeout); if (status == B_TIMED_OUT || status == B_WOULD_BLOCK) return B_OK; @@ -2800,19 +2798,6 @@ thread_block() } -/*! Blocks the current thread with a timeout. - - Acquires the scheduler lock and calls thread_block_with_timeout_locked(). - See there for more information. -*/ -status_t -thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout) -{ - InterruptsSpinLocker _(gSchedulerLock); - return thread_block_with_timeout_locked(timeoutFlags, timeout); -} - - /*! Blocks the current thread with a timeout. The thread is blocked until someone else unblock it or the specified timeout @@ -2821,7 +2806,7 @@ thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout) thread_prepare_to_block(), this function will return immediately. See thread_prepare_to_block() for more details. - The caller must hold the scheduler lock. + The caller must not hold the scheduler lock. \param thread The current thread. \param timeoutFlags The standard timeout flags: @@ -2841,10 +2826,12 @@ thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout) client code). */ status_t -thread_block_with_timeout_locked(uint32 timeoutFlags, bigtime_t timeout) +thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout) { Thread* thread = thread_get_current_thread(); + InterruptsSpinLocker locker(gSchedulerLock); + if (thread->wait.status != 1) return thread->wait.status; @@ -2871,6 +2858,8 @@ thread_block_with_timeout_locked(uint32 timeoutFlags, bigtime_t timeout) // block status_t error = thread_block_locked(thread); + locker.Unlock(); + // cancel timer, if it didn't fire if (error != B_TIMED_OUT && useTimer) cancel_timer(&thread->wait.unblock_timer); @@ -3584,11 +3573,9 @@ _user_block_thread(uint32 flags, bigtime_t timeout) thread_prepare_to_block(thread, flags, THREAD_BLOCK_TYPE_OTHER, "user"); threadLocker.Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - status_t status = thread_block_with_timeout_locked(flags, timeout); + status_t status = thread_block_with_timeout(flags, timeout); - schedulerLocker.Unlock(); threadLocker.Lock(); // Interruptions or timeouts can race with other threads unblocking us. From 3c819aaa72eb631b54bb052af527f7f2033f3646 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 4 Nov 2013 23:51:18 +0100 Subject: [PATCH 092/273] kernel: DPC: remove schedulerLocked argument --- headers/private/kernel/DPC.h | 6 ++---- src/system/kernel/DPC.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/headers/private/kernel/DPC.h b/headers/private/kernel/DPC.h index 9267910822..3c883a7a21 100644 --- a/headers/private/kernel/DPC.h +++ b/headers/private/kernel/DPC.h @@ -62,10 +62,8 @@ public: uint32 reservedSlots); void Close(bool cancelPending); - status_t Add(DPCCallback* callback, - bool schedulerLocked); - status_t Add(void (*function)(void*), void* argument, - bool schedulerLocked); + status_t Add(DPCCallback* callback); + status_t Add(void (*function)(void*), void* argument); bool Cancel(DPCCallback* callback); thread_id Thread() const diff --git a/src/system/kernel/DPC.cpp b/src/system/kernel/DPC.cpp index 7d64ca4a84..4d7261663c 100644 --- a/src/system/kernel/DPC.cpp +++ b/src/system/kernel/DPC.cpp @@ -158,7 +158,7 @@ DPCQueue::Close(bool cancelPending) status_t -DPCQueue::Add(DPCCallback* callback, bool schedulerLocked) +DPCQueue::Add(DPCCallback* callback) { // queue the callback, if the queue isn't closed already InterruptsSpinLocker locker(fLock); @@ -174,14 +174,14 @@ DPCQueue::Add(DPCCallback* callback, bool schedulerLocked) // notify the condition variable, if necessary if (wasEmpty) - fPendingCallbacksCondition.NotifyAll(schedulerLocked); + fPendingCallbacksCondition.NotifyAll(); return B_OK; } status_t -DPCQueue::Add(void (*function)(void*), void* argument, bool schedulerLocked) +DPCQueue::Add(void (*function)(void*), void* argument) { if (function == NULL) return B_BAD_VALUE; @@ -201,7 +201,7 @@ DPCQueue::Add(void (*function)(void*), void* argument, bool schedulerLocked) functionCallback->SetTo(function, argument); // add it - status_t error = Add(functionCallback, schedulerLocked); + status_t error = Add(functionCallback); if (error != B_OK) Recycle(functionCallback); From 958f6d00aad9ce038ea1b96a92e9d70099bb396b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 4 Nov 2013 23:53:20 +0100 Subject: [PATCH 093/273] kernel: Make UserEvent::Fire() work without gSchedulerLock held --- src/system/kernel/UserEvent.cpp | 50 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp index 7d890d9cf5..fb8c0b9725 100644 --- a/src/system/kernel/UserEvent.cpp +++ b/src/system/kernel/UserEvent.cpp @@ -31,22 +31,20 @@ struct SignalEvent::EventSignal : Signal { { } - bool IsInUse() const + bool MarkUsed() { - return fInUse; + return atomic_set(reinterpret_cast(&fInUse), true); } - void SetInUse(bool inUse) + void SetUnused() { - fInUse = inUse; + fInUse = false; } virtual void Handled() { // mark not-in-use - InterruptsSpinLocker schedulerLocker(gSchedulerLock); fInUse = false; - schedulerLocker.Unlock(); Signal::Handled(); } @@ -111,20 +109,22 @@ TeamSignalEvent::Create(Team* team, uint32 signalNumber, int32 signalCode, status_t TeamSignalEvent::Fire() { - // called with the scheduler lock held - if (fSignal->IsInUse()) + if (fSignal->MarkUsed()) return B_BUSY; fSignal->AcquireReference(); // one reference is transferred to send_signal_to_team_locked + + InterruptsSpinLocker locker(gSchedulerLock); status_t error = send_signal_to_team_locked(fTeam, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); if (error == B_OK) { - // Mark the signal in-use. There are situations (for certain signals), - // in which send_signal_to_team_locked() succeeds without queuing the - // signal. - fSignal->SetInUse(fSignal->IsPending()); + // There are situations (for certain signals), in which + // send_signal_to_team_locked() succeeds without queuing the signal. + if (!fSignal->IsPending()) + fSignal->SetUnused(); } + locker.Unlock(); return error; } @@ -165,20 +165,22 @@ ThreadSignalEvent::Create(Thread* thread, uint32 signalNumber, int32 signalCode, status_t ThreadSignalEvent::Fire() { - // called with the scheduler lock held - if (fSignal->IsInUse()) + if (fSignal->MarkUsed()) return B_BUSY; fSignal->AcquireReference(); // one reference is transferred to send_signal_to_team_locked + + InterruptsSpinLocker locker(gSchedulerLock); status_t error = send_signal_to_thread_locked(fThread, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); if (error == B_OK) { - // Mark the signal in-use. There are situations (for certain signals), - // in which send_signal_to_team_locked() succeeds without queuing the - // signal. - fSignal->SetInUse(fSignal->IsPending()); + // There are situations (for certain signals), in which + // send_signal_to_team_locked() succeeds without queuing the signal. + if (!fSignal->IsPending()) + fSignal->SetUnused(); } + locker.Unlock(); return error; } @@ -216,12 +218,11 @@ CreateThreadEvent::Create(const ThreadCreationAttributes& attributes) status_t CreateThreadEvent::Fire() { - if (fPendingDPC) + bool wasPending = atomic_set(reinterpret_cast(&fPendingDPC), true); + if (wasPending) return B_BUSY; - fPendingDPC = true; - - DPCQueue::DefaultQueue(B_NORMAL_PRIORITY)->Add(this, true); + DPCQueue::DefaultQueue(B_NORMAL_PRIORITY)->Add(this); return B_OK; } @@ -231,10 +232,7 @@ void CreateThreadEvent::DoDPC(DPCQueue* queue) { // We're no longer queued in the DPC queue, so we can be reused. - { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - fPendingDPC = false; - } + fPendingDPC = false; // create the thread thread_id threadID = thread_create_thread(fCreationAttributes, false); From 4824f7630b2ca9c5750f93c4daa837dfcac3059e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 5 Nov 2013 04:16:13 +0100 Subject: [PATCH 094/273] kernel: Add sequential lock implementation --- headers/os/drivers/KernelExport.h | 17 +++++++++ headers/private/kernel/smp.h | 45 ++++++++++++++++++++++++ headers/private/kernel/util/AutoLock.h | 48 ++++++++++++++++++++++++++ src/system/kernel/smp.cpp | 48 ++++++++++++++++++++++++++ 4 files changed, 158 insertions(+) diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index 98e59387f1..be17861cb6 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -40,6 +40,17 @@ typedef ulong cpu_status; # define B_SPINLOCK_IS_LOCKED(lock) (*(lock) > 0) #endif +typedef struct { + spinlock lock; + uint32 count; +} seqlock; + +#define B_SEQLOCK_INITIALIZER { B_SPINLOCK_INITIALIZER, 0 } +#define B_INITIALIZE_SEQLOCK(seqlock) do { \ + B_INITIALIZE_SPINLOCK((seqlock)->lock); \ + (seqlock)->count = 0; \ + } while (false) + /* interrupt handling support for device drivers */ typedef int32 (*interrupt_handler)(void *data); @@ -126,6 +137,12 @@ extern void restore_interrupts(cpu_status status); extern void acquire_spinlock(spinlock *lock); extern void release_spinlock(spinlock *lock); +extern bool try_acquire_write_seqlock(seqlock* lock); +extern void acquire_write_seqlock(seqlock* lock); +extern void release_write_seqlock(seqlock* lock); +extern uint32 acquire_read_seqlock(seqlock* lock); +extern bool release_read_seqlock(seqlock* lock, uint32 count); + extern status_t install_io_interrupt_handler(long interrupt_number, interrupt_handler handler, void *data, ulong flags); extern status_t remove_io_interrupt_handler(long interrupt_number, diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 90427fb825..50bdcbe4c3 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -103,4 +103,49 @@ release_spinlock_inline(spinlock* lock) #endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION +static inline bool +try_acquire_write_seqlock_inline(seqlock* lock) { + bool succeed = try_acquire_spinlock(&lock->lock); + if (succeed) + atomic_add(&lock->count, 1); + return succeed; +} + + +static inline void +acquire_write_seqlock_inline(seqlock* lock) { + acquire_spinlock(&lock->lock); + atomic_add(&lock->count, 1); +} + + +static inline void +release_write_seqlock_inline(seqlock* lock) { + atomic_add(&lock->count, 1); + release_spinlock(&lock->lock); +} + + +static inline uint32 +acquire_read_seqlock_inline(seqlock* lock) { + return atomic_get(&lock->count); +} + + +static inline bool +release_read_seqlock_inline(seqlock* lock, uint32 count) { + uint32 current = atomic_get(&lock->count); + + return count % 2 == 0 && current == count; +} + + +#define try_acquire_write_seqlock(lock) try_acquire_write_seqlock_inline(lock) +#define acquire_write_seqlock(lock) acquire_write_seqlock_inline(lock) +#define release_write_seqlock(lock) release_write_seqlock_inline(lock) +#define acquire_read_seqlock(lock) acquire_read_seqlock_inline(lock) +#define release_read_seqlock(lock, count) \ + release_read_seqlock_inline(lock, count) + + #endif /* KERNEL_SMP_H */ diff --git a/headers/private/kernel/util/AutoLock.h b/headers/private/kernel/util/AutoLock.h index 12f9b972b4..5a334fbc1a 100644 --- a/headers/private/kernel/util/AutoLock.h +++ b/headers/private/kernel/util/AutoLock.h @@ -160,6 +160,52 @@ private: typedef AutoLocker InterruptsSpinLocker; +class WriteSequentialLocking { +public: + inline bool Lock(seqlock* lockable) + { + acquire_write_seqlock(lockable); + return true; + } + + inline void Unlock(seqlock* lockable) + { + release_write_seqlock(lockable); + } +}; + +typedef AutoLocker WriteSequentialLocker; + + +class InterruptsWriteSequentialLocking { +public: + InterruptsWriteSequentialLocking() + : + fState(0) + { + } + + inline bool Lock(seqlock* lockable) + { + fState = disable_interrupts(); + acquire_write_seqlock(lockable); + return true; + } + + inline void Unlock(seqlock* lockable) + { + release_write_seqlock(lockable); + restore_interrupts(fState); + } + +private: + int fState; +}; + +typedef AutoLocker + InterruptsWriteSequentialLocker; + + class ThreadCPUPinLocking { public: inline bool Lock(Thread* thread) @@ -191,6 +237,8 @@ using BPrivate::WriteLocker; using BPrivate::InterruptsLocker; using BPrivate::SpinLocker; using BPrivate::InterruptsSpinLocker; +using BPrivate::WriteSequentialLocker; +using BPrivate::InterruptsWriteSequentialLocker; using BPrivate::ThreadCPUPinner; using BPrivate::TeamLocker; using BPrivate::ThreadLocker; diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index e28faf619c..93be92fe2e 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -45,6 +45,12 @@ #undef acquire_spinlock #undef release_spinlock +#undef try_acquire_write_seqlock +#undef acquire_write_seqlock +#undef release_write_seqlock +#undef acquire_read_seqlock +#undef release_read_seqlock + #define MSG_POOL_SIZE (SMP_MAX_CPUS * 4) @@ -512,6 +518,48 @@ release_spinlock(spinlock *lock) } +bool +try_acquire_write_seqlock(seqlock* lock) { + bool succeed = try_acquire_spinlock(&lock->lock); + if (succeed) + atomic_add(&lock->count, 1); + return succeed; +} + + +void +acquire_write_seqlock(seqlock* lock) { + acquire_spinlock(&lock->lock); + atomic_add(&lock->count, 1); +} + + +void +release_write_seqlock(seqlock* lock) { + atomic_add(&lock->count, 1); + release_spinlock(&lock->lock); +} + + +uint32 +acquire_read_seqlock(seqlock* lock) { + return atomic_get(&lock->count); +} + + +bool +release_read_seqlock(seqlock* lock, uint32 count) { + uint32 current = atomic_get(&lock->count); + + if (count % 2 == 1 || current != count) { + PAUSE(); + return false; + } + + return true; +} + + /*! Finds a free message and gets it. NOTE: has side effect of disabling interrupts return value is the former interrupt state From f4b088a992fefefc87aae57643aca05d81826cca Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 5 Nov 2013 04:16:32 +0100 Subject: [PATCH 095/273] kernel: Protect UserTimers with sUserTimerLock --- headers/private/kernel/UserEvent.h | 2 +- headers/private/kernel/UserTimer.h | 3 + src/system/kernel/UserEvent.cpp | 41 +++---- src/system/kernel/UserTimer.cpp | 176 ++++++++++++++++++----------- 4 files changed, 136 insertions(+), 86 deletions(-) diff --git a/headers/private/kernel/UserEvent.h b/headers/private/kernel/UserEvent.h index 1b8ceb70b2..34086bc57e 100644 --- a/headers/private/kernel/UserEvent.h +++ b/headers/private/kernel/UserEvent.h @@ -92,7 +92,7 @@ private: private: ThreadCreationAttributes fCreationAttributes; char fThreadName[B_OS_NAME_LENGTH]; - bool fPendingDPC; + int32 fPendingDPC; }; diff --git a/headers/private/kernel/UserTimer.h b/headers/private/kernel/UserTimer.h index 2f439c1b88..026aa55575 100644 --- a/headers/private/kernel/UserTimer.h +++ b/headers/private/kernel/UserTimer.h @@ -54,6 +54,8 @@ protected: inline void UpdatePeriodicStartTime(); inline void CheckPeriodicOverrun(bigtime_t now); + inline void CancelTimer(); + protected: int32 fID; timer fTimer; @@ -62,6 +64,7 @@ protected: bigtime_t fInterval; uint32 fOverrunCount; bool fScheduled; // fTimer scheduled + int32 fSkip; }; diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp index fb8c0b9725..16f5aec482 100644 --- a/src/system/kernel/UserEvent.cpp +++ b/src/system/kernel/UserEvent.cpp @@ -27,30 +27,33 @@ struct SignalEvent::EventSignal : Signal { pid_t sendingProcess) : Signal(number, signalCode, errorCode, sendingProcess), - fInUse(false) + fInUse(0) { } bool MarkUsed() { - return atomic_set(reinterpret_cast(&fInUse), true); + return atomic_set(&fInUse, 1) != 0; } void SetUnused() { - fInUse = false; + fInUse = 0; } virtual void Handled() { // mark not-in-use - fInUse = false; + { + InterruptsSpinLocker _(gSchedulerLock); + fInUse = 0; + } Signal::Handled(); } private: - bool fInUse; + int32 fInUse; }; @@ -118,12 +121,10 @@ TeamSignalEvent::Fire() InterruptsSpinLocker locker(gSchedulerLock); status_t error = send_signal_to_team_locked(fTeam, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); - if (error == B_OK) { - // There are situations (for certain signals), in which - // send_signal_to_team_locked() succeeds without queuing the signal. - if (!fSignal->IsPending()) - fSignal->SetUnused(); - } + // There are situations (for certain signals), in which + // send_signal_to_team_locked() succeeds without queuing the signal. + if (error != B_OK || !fSignal->IsPending()) + fSignal->SetUnused(); locker.Unlock(); return error; @@ -165,6 +166,8 @@ ThreadSignalEvent::Create(Thread* thread, uint32 signalNumber, int32 signalCode, status_t ThreadSignalEvent::Fire() { + dprintf("THREAD\n"); + if (fSignal->MarkUsed()) return B_BUSY; @@ -174,12 +177,10 @@ ThreadSignalEvent::Fire() InterruptsSpinLocker locker(gSchedulerLock); status_t error = send_signal_to_thread_locked(fThread, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); - if (error == B_OK) { - // There are situations (for certain signals), in which - // send_signal_to_team_locked() succeeds without queuing the signal. - if (!fSignal->IsPending()) - fSignal->SetUnused(); - } + // There are situations (for certain signals), in which + // send_signal_to_team_locked() succeeds without queuing the signal. + if (error != B_OK || !fSignal->IsPending()) + fSignal->SetUnused(); locker.Unlock(); return error; @@ -192,7 +193,7 @@ ThreadSignalEvent::Fire() CreateThreadEvent::CreateThreadEvent(const ThreadCreationAttributes& attributes) : fCreationAttributes(attributes), - fPendingDPC(false) + fPendingDPC(0) { // attributes.name is a pointer to a temporary buffer. Copy the name into // our own buffer and replace the name pointer. @@ -218,7 +219,7 @@ CreateThreadEvent::Create(const ThreadCreationAttributes& attributes) status_t CreateThreadEvent::Fire() { - bool wasPending = atomic_set(reinterpret_cast(&fPendingDPC), true); + bool wasPending = atomic_set(&fPendingDPC, 1) != 0; if (wasPending) return B_BUSY; @@ -232,7 +233,7 @@ void CreateThreadEvent::DoDPC(DPCQueue* queue) { // We're no longer queued in the DPC queue, so we can be reused. - fPendingDPC = false; + fPendingDPC = 0; // create the thread thread_id threadID = thread_create_thread(fCreationAttributes, false); diff --git a/src/system/kernel/UserTimer.cpp b/src/system/kernel/UserTimer.cpp index 70e6082a2b..05208cf1f8 100644 --- a/src/system/kernel/UserTimer.cpp +++ b/src/system/kernel/UserTimer.cpp @@ -29,6 +29,8 @@ static const bigtime_t kMinPeriodicTimerInterval = 100; static RealTimeUserTimerList sAbsoluteRealTimeTimers; static spinlock sAbsoluteRealTimeTimersLock = B_SPINLOCK_INITIALIZER; +static seqlock sUserTimerLock = B_SEQLOCK_INITIALIZER; + // #pragma mark - TimerLocker @@ -116,7 +118,8 @@ UserTimer::UserTimer() fNextTime(0), fInterval(0), fOverrunCount(0), - fScheduled(false) + fScheduled(false), + fSkip(0) { // mark the timer unused fTimer.user_data = this; @@ -191,8 +194,22 @@ UserTimer::Cancel() /*static*/ int32 UserTimer::HandleTimerHook(struct timer* timer) { - InterruptsSpinLocker _(gSchedulerLock); - ((UserTimer*)timer->user_data)->HandleTimer(); + UserTimer* userTimer = reinterpret_cast(timer->user_data); + + InterruptsLocker _; + + bool locked = false; + while (!locked && atomic_get(&userTimer->fSkip) == 0) { + locked = try_acquire_write_seqlock(&sUserTimerLock); + if (!locked) + PAUSE(); + } + + if (locked) { + userTimer->HandleTimer(); + release_write_seqlock(&sUserTimerLock); + } + return B_HANDLED_INTERRUPT; } @@ -219,7 +236,7 @@ UserTimer::HandleTimer() /*! Updates the start time for a periodic timer after it expired, enforcing sanity limits and updating \c fOverrunCount, if necessary. - The caller must not hold the scheduler lock. + The caller must not hold \c sUserTimerLock. */ void UserTimer::UpdatePeriodicStartTime() @@ -243,7 +260,7 @@ UserTimer::UpdatePeriodicStartTime() /*! Checks whether the timer start time lies too much in the past and, if so, adjusts it and updates \c fOverrunCount. - The caller must not hold the scheduler lock. + The caller must not hold \c sUserTimerLock. \param now The current time. */ @@ -265,6 +282,17 @@ UserTimer::CheckPeriodicOverrun(bigtime_t now) } +void +UserTimer::CancelTimer() +{ + ASSERT(fScheduled); + + fSkip = 1; + cancel_timer(&fTimer); + fSkip = 0; +} + + // #pragma mark - SystemTimeUserTimer @@ -272,14 +300,14 @@ void SystemTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsWriteSequentialLocker locker(sUserTimerLock); // get the current time bigtime_t now = system_time(); // Cancel the old timer, if still scheduled, and get the previous values. if (fScheduled) { - cancel_timer(&fTimer); + CancelTimer(); _oldRemainingTime = fNextTime - now; _oldInterval = fInterval; @@ -309,17 +337,20 @@ void SystemTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, uint32& _overrunCount) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + uint32 count; + do { + count = acquire_read_seqlock(&sUserTimerLock); - if (fScheduled) { - _remainingTime = fNextTime - system_time(); - _interval = fInterval; - } else { - _remainingTime = B_INFINITE_TIMEOUT; - _interval = 0; - } + if (fScheduled) { + _remainingTime = fNextTime - system_time(); + _interval = fInterval; + } else { + _remainingTime = B_INFINITE_TIMEOUT; + _interval = 0; + } - _overrunCount = fOverrunCount; + _overrunCount = fOverrunCount; + } while (!release_read_seqlock(&sUserTimerLock, count)); } @@ -338,7 +369,7 @@ SystemTimeUserTimer::HandleTimer() /*! Schedules the kernel timer. - The caller must hold the scheduler lock. + The caller must hold \c sUserTimerLock. \param now The current system time to be used. \param checkPeriodicOverrun If \c true, calls CheckPeriodicOverrun() first, @@ -371,14 +402,14 @@ void RealTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsWriteSequentialLocker locker(sUserTimerLock); // get the current time bigtime_t now = system_time(); // Cancel the old timer, if still scheduled, and get the previous values. if (fScheduled) { - cancel_timer(&fTimer); + CancelTimer(); _oldRemainingTime = fNextTime - now; _oldInterval = fInterval; @@ -424,7 +455,7 @@ RealTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, /*! Called when the real-time clock has been changed. - The caller must hold the scheduler lock. Optionally the caller may also + The caller must hold \c sUserTimerLock. Optionally the caller may also hold \c sAbsoluteRealTimeTimersLock. */ void @@ -439,7 +470,7 @@ RealTimeUserTimer::TimeWarped() return; // cancel the kernel timer and reschedule it - cancel_timer(&fTimer); + CancelTimer(); fNextTime += oldRealTimeOffset - fRealTimeOffset; @@ -481,6 +512,7 @@ void TeamTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { + InterruptsWriteSequentialLocker locker(sUserTimerLock); InterruptsSpinLocker schedulerLocker(gSchedulerLock); // get the current time, but only if needed @@ -490,7 +522,7 @@ TeamTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, // Cancel the old timer, if still scheduled, and get the previous values. if (fTeam != NULL) { if (fScheduled) { - cancel_timer(&fTimer); + CancelTimer(); fScheduled = false; } @@ -539,23 +571,27 @@ void TeamTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, uint32& _overrunCount) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + uint32 count; + do { + count = acquire_read_seqlock(&sUserTimerLock); - if (fTeam != NULL) { - _remainingTime = fNextTime - fTeam->CPUTime(false); - _interval = fInterval; - } else { - _remainingTime = B_INFINITE_TIMEOUT; - _interval = 0; - } + if (fTeam != NULL) { + InterruptsSpinLocker schedulerLocker(gSchedulerLock); + _remainingTime = fNextTime - fTeam->CPUTime(false); + _interval = fInterval; + } else { + _remainingTime = B_INFINITE_TIMEOUT; + _interval = 0; + } - _overrunCount = fOverrunCount; + _overrunCount = fOverrunCount; + } while (!release_read_seqlock(&sUserTimerLock, count)); } /*! Deactivates the timer, if it is activated. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. */ void TeamTimeUserTimer::Deactivate() @@ -565,7 +601,7 @@ TeamTimeUserTimer::Deactivate() // unschedule, if scheduled if (fScheduled) { - cancel_timer(&fTimer); + CancelTimer(); fScheduled = false; } @@ -583,7 +619,7 @@ TeamTimeUserTimer::Deactivate() was just set. Schedules a kernel timer for the remaining time, respectively cancels it. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. \param unscheduledThread If not \c NULL, this is the thread that is currently running and which is in the process of being unscheduled. @@ -610,7 +646,7 @@ TeamTimeUserTimer::Update(Thread* unscheduledThread) /*! Called when the team's CPU time clock which this timer refers to has been set. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. \param changedBy The value by which the clock has changed. */ @@ -653,7 +689,7 @@ TeamTimeUserTimer::HandleTimer() /*! Schedules/cancels the kernel timer as necessary. \c fRunningThreads must be up-to-date. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. \param unscheduling \c true, when the current thread is in the process of being unscheduled. @@ -663,7 +699,7 @@ TeamTimeUserTimer::_Update(bool unscheduling) { // unschedule the kernel timer, if scheduled if (fScheduled) - cancel_timer(&fTimer); + CancelTimer(); // if no more threads are running, we're done if (fRunningThreads == 0) { @@ -720,6 +756,7 @@ void TeamUserTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { + InterruptsWriteSequentialLocker locker(sUserTimerLock); InterruptsSpinLocker schedulerLocker(gSchedulerLock); // get the current time, but only if needed @@ -771,23 +808,27 @@ void TeamUserTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, uint32& _overrunCount) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + uint32 count; + do { + count = acquire_read_seqlock(&sUserTimerLock); - if (fTeam != NULL) { - _remainingTime = fNextTime - fTeam->UserCPUTime(); - _interval = fInterval; - } else { - _remainingTime = B_INFINITE_TIMEOUT; - _interval = 0; - } + if (fTeam != NULL) { + InterruptsSpinLocker schedulerLocker(gSchedulerLock); + _remainingTime = fNextTime - fTeam->UserCPUTime(); + _interval = fInterval; + } else { + _remainingTime = B_INFINITE_TIMEOUT; + _interval = 0; + } - _overrunCount = fOverrunCount; + _overrunCount = fOverrunCount; + } while (!release_read_seqlock(&sUserTimerLock, count)); } /*! Deactivates the timer, if it is activated. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. */ void TeamUserTimeUserTimer::Deactivate() @@ -804,7 +845,7 @@ TeamUserTimeUserTimer::Deactivate() /*! Checks whether the timer is up, firing an event, if so. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. */ void TeamUserTimeUserTimer::Check() @@ -857,6 +898,7 @@ void ThreadTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { + InterruptsWriteSequentialLocker locker(sUserTimerLock); InterruptsSpinLocker schedulerLocker(gSchedulerLock); // get the current time, but only if needed @@ -866,7 +908,7 @@ ThreadTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, // Cancel the old timer, if still scheduled, and get the previous values. if (fThread != NULL) { if (fScheduled) { - cancel_timer(&fTimer); + CancelTimer(); fScheduled = false; } @@ -916,23 +958,27 @@ void ThreadTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, uint32& _overrunCount) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + uint32 count; + do { + count = acquire_read_seqlock(&sUserTimerLock); - if (fThread != NULL) { - _remainingTime = fNextTime - fThread->CPUTime(false); - _interval = fInterval; - } else { - _remainingTime = B_INFINITE_TIMEOUT; - _interval = 0; - } + if (fThread != NULL) { + InterruptsSpinLocker schedulerLocker(gSchedulerLock); + _remainingTime = fNextTime - fThread->CPUTime(false); + _interval = fInterval; + } else { + _remainingTime = B_INFINITE_TIMEOUT; + _interval = 0; + } - _overrunCount = fOverrunCount; + _overrunCount = fOverrunCount; + } while (!release_read_seqlock(&sUserTimerLock, count)); } /*! Deactivates the timer, if it is activated. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. */ void ThreadTimeUserTimer::Deactivate() @@ -942,7 +988,7 @@ ThreadTimeUserTimer::Deactivate() // unschedule, if scheduled if (fScheduled) { - cancel_timer(&fTimer); + CancelTimer(); fScheduled = false; } @@ -959,7 +1005,7 @@ ThreadTimeUserTimer::Deactivate() scheduled, or, when the timer was just set and the thread is already running. Schedules a kernel timer for the remaining time. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. */ void ThreadTimeUserTimer::Start() @@ -997,7 +1043,7 @@ ThreadTimeUserTimer::Start() Called when the thread whose CPU time is referred to by the timer is unscheduled, or, when the timer is canceled. - The caller must hold the scheduler lock. + The caller must hold \c sUserTimerLock. */ void ThreadTimeUserTimer::Stop() @@ -1008,7 +1054,7 @@ ThreadTimeUserTimer::Stop() ASSERT(fScheduled); // cancel the kernel timer - cancel_timer(&fTimer); + CancelTimer(); fScheduled = false; // TODO: To avoid odd race conditions, we should check the current time of @@ -1020,7 +1066,7 @@ ThreadTimeUserTimer::Stop() /*! Called when the team's CPU time clock which this timer refers to has been set. - The caller must hold the scheduler lock. + The caller must hold the scheduler lock and \c sUserTimerLock. \param changedBy The value by which the clock has changed. */ @@ -1480,7 +1526,7 @@ void user_timer_real_time_clock_changed() { // we need to update all absolute real-time timers - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsWriteSequentialLocker locker(sUserTimerLock); SpinLocker globalListLocker(sAbsoluteRealTimeTimersLock); for (RealTimeUserTimerList::Iterator it From e7dba861fd89792538442cc1b01584f7dabbd4a9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 5 Nov 2013 20:28:25 +0100 Subject: [PATCH 096/273] kernel: User{Event, Timer}: Use atomic access where necessary --- src/system/kernel/UserEvent.cpp | 19 ++++++++----------- src/system/kernel/UserTimer.cpp | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp index 16f5aec482..097ea2b07c 100644 --- a/src/system/kernel/UserEvent.cpp +++ b/src/system/kernel/UserEvent.cpp @@ -38,16 +38,13 @@ struct SignalEvent::EventSignal : Signal { void SetUnused() { - fInUse = 0; + // mark not-in-use + atomic_set(&fInUse, 0); } virtual void Handled() { - // mark not-in-use - { - InterruptsSpinLocker _(gSchedulerLock); - fInUse = 0; - } + SetUnused(); Signal::Handled(); } @@ -121,11 +118,12 @@ TeamSignalEvent::Fire() InterruptsSpinLocker locker(gSchedulerLock); status_t error = send_signal_to_team_locked(fTeam, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); + locker.Unlock(); + // There are situations (for certain signals), in which // send_signal_to_team_locked() succeeds without queuing the signal. if (error != B_OK || !fSignal->IsPending()) fSignal->SetUnused(); - locker.Unlock(); return error; } @@ -166,8 +164,6 @@ ThreadSignalEvent::Create(Thread* thread, uint32 signalNumber, int32 signalCode, status_t ThreadSignalEvent::Fire() { - dprintf("THREAD\n"); - if (fSignal->MarkUsed()) return B_BUSY; @@ -177,11 +173,12 @@ ThreadSignalEvent::Fire() InterruptsSpinLocker locker(gSchedulerLock); status_t error = send_signal_to_thread_locked(fThread, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); + locker.Unlock(); + // There are situations (for certain signals), in which // send_signal_to_team_locked() succeeds without queuing the signal. if (error != B_OK || !fSignal->IsPending()) fSignal->SetUnused(); - locker.Unlock(); return error; } @@ -233,7 +230,7 @@ void CreateThreadEvent::DoDPC(DPCQueue* queue) { // We're no longer queued in the DPC queue, so we can be reused. - fPendingDPC = 0; + atomic_set(&fPendingDPC, 0); // create the thread thread_id threadID = thread_create_thread(fCreationAttributes, false); diff --git a/src/system/kernel/UserTimer.cpp b/src/system/kernel/UserTimer.cpp index 05208cf1f8..6c04125774 100644 --- a/src/system/kernel/UserTimer.cpp +++ b/src/system/kernel/UserTimer.cpp @@ -287,9 +287,9 @@ UserTimer::CancelTimer() { ASSERT(fScheduled); - fSkip = 1; + atomic_set(&fSkip, 1); cancel_timer(&fTimer); - fSkip = 0; + atomic_set(&fSkip, 0); } From 077c84eb27b25430428d356f3d13afabc0cc0d13 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 5 Nov 2013 22:32:59 +0100 Subject: [PATCH 097/273] kernel: atomic_*() functions rework * No need for the atomically changed variables to be declared as volatile. * Drop support for atomically getting and setting unaligned data. * Introduce atomic_get_and_set[64]() which works the same as atomic_set[64]() used to. atomic_set[64]() does not return the previous value anymore. --- headers/build/HaikuBuildCompatibility.h | 20 +-- headers/os/drivers/KernelExport.h | 8 +- headers/os/support/String.h | 4 +- headers/os/support/SupportDefs.h | 35 ++--- headers/private/firewire/fwglue.h | 2 +- headers/private/fs_shell/fssh_api_wrapper.h | 1 + headers/private/fs_shell/fssh_atomic.h | 26 ++-- headers/private/kernel/smp.h | 12 +- headers/private/kernel/util/atomic.h | 27 +++- headers/private/libroot/pthread_private.h | 4 +- headers/private/package/TempfileManager.h | 2 +- headers/private/shared/Referenceable.h | 2 +- headers/private/shared/WeakReferenceable.h | 2 +- headers/private/shared/locks.h | 2 +- headers/private/support/StringPrivate.h | 4 +- src/add-ons/kernel/bus_managers/ps2/ps2_dev.h | 2 +- .../drivers/input/usb_hid/HIDDevice.cpp | 2 +- .../drivers/power/acpi_battery/acpi_battery.h | 2 +- .../enhanced_speedstep/enhanced_speedstep.cpp | 2 +- .../enhanced_speedstep/enhanced_speedstep.h | 2 +- .../kernel/file_systems/nfs4/FileSystem.h | 2 +- .../network/datalink_protocols/arp/arp.cpp | 2 +- .../ipv6_datagram/ipv6_datagram.cpp | 2 +- .../kernel/network/stack/net_buffer.cpp | 12 +- .../imap/imap_lib/IMAPMailbox.h | 2 +- .../imap/imap_lib/IMAPProtocol.h | 2 +- src/apps/activitymonitor/ActivityView.h | 2 +- src/apps/powerstatus/DriverInterface.h | 2 +- src/build/libroot/atomic.cpp | 38 +++-- src/kits/support/String.cpp | 4 +- src/system/kernel/UserEvent.cpp | 4 +- src/system/kernel/arch/x86/arch_cpu.cpp | 6 +- .../x86/paging/32bit/X86PagingMethod32Bit.h | 2 +- .../x86/paging/64bit/X86PagingMethod64Bit.h | 2 +- .../arch/x86/paging/pae/X86PagingMethodPAE.h | 2 +- src/system/kernel/fs/EntryCache.cpp | 3 +- src/system/kernel/fs/EntryCache.h | 6 +- src/system/kernel/fs/unused_vnodes.h | 12 +- src/system/kernel/posix/xsi_message_queue.cpp | 6 +- src/system/kernel/smp.cpp | 35 ++--- src/system/libroot/os/arch/x86/atomic.S | 144 ++++++++++++------ src/system/libroot/os/arch/x86_64/atomic.S | 76 +++++---- src/system/libroot/os/locks/init_once.cpp | 4 +- .../libroot/posix/pthread/pthread_key.cpp | 2 +- .../libroot/posix/pthread/pthread_once.cpp | 8 +- src/tools/fs_shell/atomic.cpp | 64 +++++--- src/tools/fs_shell/vfs.cpp | 8 +- 47 files changed, 360 insertions(+), 253 deletions(-) diff --git a/headers/build/HaikuBuildCompatibility.h b/headers/build/HaikuBuildCompatibility.h index 009c77f1d5..38b5197fb8 100644 --- a/headers/build/HaikuBuildCompatibility.h +++ b/headers/build/HaikuBuildCompatibility.h @@ -118,17 +118,19 @@ struct media_node; extern "C" { #endif -extern int32 atomic_set(vint32 *value, int32 newValue); -extern int32 atomic_test_and_set(vint32 *value, int32 newValue, +extern void atomic_set(int32* value, int32 newValue); +extern int32 atomic_get_and_set(int32* value, int32 newValue); +extern int32 atomic_test_and_set(int32 *value, int32 newValue, int32 testAgainst); -extern int32 atomic_get(vint32 *value); -extern int64 atomic_set64(vint64 *value, int64 newValue); -extern int64 atomic_test_and_set64(vint64 *value, int64 newValue, +extern int32 atomic_get(int32 *value); +extern void atomic_set64(int64* value, int64 newValue); +extern int64 atomic_get_and_set64(int64* value, int64 newValue); +extern int64 atomic_test_and_set64(int64 *value, int64 newValue, int64 testAgainst); -extern int64 atomic_get64(vint64 *value); -extern int64 atomic_add64(vint64 *value, int64 addValue); -extern int64 atomic_and64(vint64 *value, int64 andValue); -extern int64 atomic_or64(vint64 *value, int64 orValue); +extern int64 atomic_get64(int64 *value); +extern int64 atomic_add64(int64 *value, int64 addValue); +extern int64 atomic_and64(int64 *value, int64 andValue); +extern int64 atomic_or64(int64 *value, int64 orValue); extern size_t strnlen(const char *string, size_t count); diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index be17861cb6..d4168557dd 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -20,9 +20,9 @@ typedef ulong cpu_status; #if B_DEBUG_SPINLOCK_CONTENTION typedef struct { - vint32 lock; - vint32 count_low; - vint32 count_high; + int32 lock; + int32 count_low; + int32 count_high; } spinlock; # define B_SPINLOCK_INITIALIZER { 0, 0, 0 } @@ -33,7 +33,7 @@ typedef ulong cpu_status; } while (false) # define B_SPINLOCK_IS_LOCKED(spinlock) ((spinlock)->lock > 0) #else - typedef vint32 spinlock; + typedef int32 spinlock; # define B_SPINLOCK_INITIALIZER 0 # define B_INITIALIZE_SPINLOCK(lock) do { *(lock) = 0; } while (false) diff --git a/headers/os/support/String.h b/headers/os/support/String.h index e33fbc0540..7ca3b64442 100644 --- a/headers/os/support/String.h +++ b/headers/os/support/String.h @@ -396,8 +396,8 @@ private: int32 withLength); private: - vint32& _ReferenceCount(); - const vint32& _ReferenceCount() const; + int32& _ReferenceCount(); + const int32& _ReferenceCount() const; bool _IsShareable() const; void _FreePrivateData(); diff --git a/headers/os/support/SupportDefs.h b/headers/os/support/SupportDefs.h index 7cc5aaf67e..aae8002000 100644 --- a/headers/os/support/SupportDefs.h +++ b/headers/os/support/SupportDefs.h @@ -196,19 +196,21 @@ extern "C" { #endif /* Atomic functions; previous value is returned */ -extern int32 atomic_set(vint32 *value, int32 newValue); -extern int32 atomic_test_and_set(vint32 *value, int32 newValue, int32 testAgainst); -extern int32 atomic_add(vint32 *value, int32 addValue); -extern int32 atomic_and(vint32 *value, int32 andValue); -extern int32 atomic_or(vint32 *value, int32 orValue); -extern int32 atomic_get(vint32 *value); +extern void atomic_set(int32* value, int32 newValue); +extern int32 atomic_get_and_set(int32* value, int32 newValue); +extern int32 atomic_test_and_set(int32 *value, int32 newValue, int32 testAgainst); +extern int32 atomic_add(int32 *value, int32 addValue); +extern int32 atomic_and(int32 *value, int32 andValue); +extern int32 atomic_or(int32 *value, int32 orValue); +extern int32 atomic_get(int32 *value); -extern int64 atomic_set64(vint64 *value, int64 newValue); -extern int64 atomic_test_and_set64(vint64 *value, int64 newValue, int64 testAgainst); -extern int64 atomic_add64(vint64 *value, int64 addValue); -extern int64 atomic_and64(vint64 *value, int64 andValue); -extern int64 atomic_or64(vint64 *value, int64 orValue); -extern int64 atomic_get64(vint64 *value); +extern void atomic_set64(int64* value, int64 newValue); +extern int64 atomic_get_and_set64(int64* value, int64 newValue); +extern int64 atomic_test_and_set64(int64 *value, int64 newValue, int64 testAgainst); +extern int64 atomic_add64(int64 *value, int64 addValue); +extern int64 atomic_and64(int64 *value, int64 andValue); +extern int64 atomic_or64(int64 *value, int64 orValue); +extern int64 atomic_get64(int64 *value); /* Other stuff */ extern void* get_stack_frame(void); @@ -240,15 +242,6 @@ extern void* get_stack_frame(void); __sync_fetch_and_and(valuePointer, andValue) #define atomic_or(valuePointer, orValue) \ __sync_fetch_and_or(valuePointer, orValue) -#define atomic_get(valuePointer) \ - __sync_fetch_and_or(valuePointer, 0) - // No equivalent to atomic_get(). We simulate it via atomic or. On most - // (all?) 32+ bit architectures aligned 32 bit reads will be atomic anyway, - // though. - -// Note: No equivalent for atomic_set(). It could be simulated by a -// get + atomic test and set loop, but calling the atomic_set() implementation -// might be faster. #endif // B_USE_BUILTIN_ATOMIC_FUNCTIONS && __GNUC__ >= 4 diff --git a/headers/private/firewire/fwglue.h b/headers/private/firewire/fwglue.h index 896224093f..453ad32465 100644 --- a/headers/private/firewire/fwglue.h +++ b/headers/private/firewire/fwglue.h @@ -30,7 +30,7 @@ typedef uint32_t bus_addr_t; typedef uint32_t bus_size_t; -#define atomic_readandclear_int(ptr) atomic_set((int32 *)(ptr), 0) +#define atomic_readandclear_int(ptr) atomic_get_and_set((int32*)(ptr), 0) #define atomic_set_int(ptr, value) atomic_or((int32 *)(ptr), value) #define mtx_lock mutex_lock diff --git a/headers/private/fs_shell/fssh_api_wrapper.h b/headers/private/fs_shell/fssh_api_wrapper.h index edb40a9458..679e360dcc 100644 --- a/headers/private/fs_shell/fssh_api_wrapper.h +++ b/headers/private/fs_shell/fssh_api_wrapper.h @@ -50,6 +50,7 @@ // #pragma mark - fssh_atomic.h #define atomic_set fssh_atomic_set +#define atomic_get_and_Set fssh_atomic_get_and_set #define atomic_test_and_set fssh_atomic_test_and_set #define atomic_add fssh_atomic_add #define atomic_and fssh_atomic_and diff --git a/headers/private/fs_shell/fssh_atomic.h b/headers/private/fs_shell/fssh_atomic.h index df307a46f1..90fba9ca7f 100644 --- a/headers/private/fs_shell/fssh_atomic.h +++ b/headers/private/fs_shell/fssh_atomic.h @@ -15,21 +15,23 @@ extern "C" { #endif -int32_t fssh_atomic_set(vint32_t *value, int32_t newValue); -int32_t fssh_atomic_test_and_set(vint32_t *value, int32_t newValue, +void fssh_atomic_set(int32_t* value, int32_t newValue); +int32_t fssh_atomic_get_and_set(int32_t* value, int32_t newValue); +int32_t fssh_atomic_test_and_set(int32_t *value, int32_t newValue, int32_t testAgainst); -int32_t fssh_atomic_add(vint32_t *value, int32_t addValue); -int32_t fssh_atomic_and(vint32_t *value, int32_t andValue); -int32_t fssh_atomic_or(vint32_t *value, int32_t orValue); -int32_t fssh_atomic_get(vint32_t *value); +int32_t fssh_atomic_add(int32_t *value, int32_t addValue); +int32_t fssh_atomic_and(int32_t *value, int32_t andValue); +int32_t fssh_atomic_or(int32_t *value, int32_t orValue); +int32_t fssh_atomic_get(int32_t *value); -int64_t fssh_atomic_set64(vint64_t *value, int64_t newValue); -int64_t fssh_atomic_test_and_set64(vint64_t *value, int64_t newValue, +void fssh_atomic_set64(int64_t* value, int64_t newValue); +int64_t fssh_atomic_get_and_set64(int64_t* value, int64_t newValue); +int64_t fssh_atomic_test_and_set64(int64_t *value, int64_t newValue, int64_t testAgainst); -int64_t fssh_atomic_add64(vint64_t *value, int64_t addValue); -int64_t fssh_atomic_and64(vint64_t *value, int64_t andValue); -int64_t fssh_atomic_or64(vint64_t *value, int64_t orValue); -int64_t fssh_atomic_get64(vint64_t *value); +int64_t fssh_atomic_add64(int64_t *value, int64_t addValue); +int64_t fssh_atomic_and64(int64_t *value, int64_t andValue); +int64_t fssh_atomic_or64(int64_t *value, int64_t orValue); +int64_t fssh_atomic_get64(int64_t *value); #ifdef __cplusplus } diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 50bdcbe4c3..d20ee28e0a 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -47,7 +47,7 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); status_t smp_init_post_generic_syscalls(void); bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); void smp_wake_up_non_boot_cpus(void); -void smp_cpu_rendezvous(volatile uint32 *var, int current_cpu); +void smp_cpu_rendezvous(uint32 *var, int current_cpu); void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, @@ -107,7 +107,7 @@ static inline bool try_acquire_write_seqlock_inline(seqlock* lock) { bool succeed = try_acquire_spinlock(&lock->lock); if (succeed) - atomic_add(&lock->count, 1); + atomic_add((int32*)&lock->count, 1); return succeed; } @@ -115,26 +115,26 @@ try_acquire_write_seqlock_inline(seqlock* lock) { static inline void acquire_write_seqlock_inline(seqlock* lock) { acquire_spinlock(&lock->lock); - atomic_add(&lock->count, 1); + atomic_add((int32*)&lock->count, 1); } static inline void release_write_seqlock_inline(seqlock* lock) { - atomic_add(&lock->count, 1); + atomic_add((int32*)&lock->count, 1); release_spinlock(&lock->lock); } static inline uint32 acquire_read_seqlock_inline(seqlock* lock) { - return atomic_get(&lock->count); + return atomic_get((int32*)&lock->count); } static inline bool release_read_seqlock_inline(seqlock* lock, uint32 count) { - uint32 current = atomic_get(&lock->count); + uint32 current = atomic_get((int32*)&lock->count); return count % 2 == 0 && current == count; } diff --git a/headers/private/kernel/util/atomic.h b/headers/private/kernel/util/atomic.h index 510df65bec..14725492c8 100644 --- a/headers/private/kernel/util/atomic.h +++ b/headers/private/kernel/util/atomic.h @@ -18,22 +18,34 @@ atomic_pointer_test_and_set(PointerType** _pointer, const PointerType* set, const PointerType* test) { #if LONG_MAX == INT_MAX - return (PointerType*)atomic_test_and_set((vint32*)_pointer, (int32)set, + return (PointerType*)atomic_test_and_set((int32*)_pointer, (int32)set, (int32)test); #else - return (PointerType*)atomic_test_and_set64((vint64*)_pointer, (int64)set, + return (PointerType*)atomic_test_and_set64((int64*)_pointer, (int64)set, (int64)test); #endif } template PointerType* -atomic_pointer_set(PointerType** _pointer, const PointerType* set) +atomic_pointer_get_and_set(PointerType** _pointer, const PointerType* set) { #if LONG_MAX == INT_MAX - return (PointerType*)atomic_set((vint32*)_pointer, (int32)set); + return (PointerType*)atomic_get_and_set((int32*)_pointer, (int32)set); #else - return (PointerType*)atomic_set64((vint64*)_pointer, (int64)set); + return (PointerType*)atomic_get_and_set64((int64*)_pointer, (int64)set); +#endif +} + + +template void +atomic_pointer_set(PointerType** _pointer, const PointerType* set) +{ + ASSERT((addr_t(_pointer) & (sizeof(PointerType*) - 1)) == 0); +#if LONG_MAX == INT_MAX + atomic_set((int32*)_pointer, (int32)set); +#else + atomic_set64((int64*)_pointer, (int64)set); #endif } @@ -41,10 +53,11 @@ atomic_pointer_set(PointerType** _pointer, const PointerType* set) template PointerType* atomic_pointer_get(PointerType** _pointer) { + ASSERT((addr_t(_pointer) & (sizeof(PointerType*) - 1)) == 0); #if LONG_MAX == INT_MAX - return (PointerType*)atomic_get((vint32*)_pointer); + return (PointerType*)atomic_get((int32*)_pointer); #else - return (PointerType*)atomic_get64((vint64*)_pointer); + return (PointerType*)atomic_get64((int64*)_pointer); #endif } diff --git a/headers/private/libroot/pthread_private.h b/headers/private/libroot/pthread_private.h index 1ddc00c6be..659ac7b832 100644 --- a/headers/private/libroot/pthread_private.h +++ b/headers/private/libroot/pthread_private.h @@ -50,12 +50,12 @@ typedef struct _pthread_rwlockattr { typedef void (*pthread_key_destructor)(void *data); struct pthread_key { - vint32 sequence; + int32 sequence; pthread_key_destructor destructor; }; struct pthread_key_data { - vint32 sequence; + int32 sequence; void *value; }; diff --git a/headers/private/package/TempfileManager.h b/headers/private/package/TempfileManager.h index e53ebadaf6..3d118b278b 100644 --- a/headers/private/package/TempfileManager.h +++ b/headers/private/package/TempfileManager.h @@ -31,7 +31,7 @@ private: private: BDirectory fBaseDirectory; - vint32 fNextNumber; + int32 fNextNumber; }; diff --git a/headers/private/shared/Referenceable.h b/headers/private/shared/Referenceable.h index ed9eff8663..0c7b2980df 100644 --- a/headers/private/shared/Referenceable.h +++ b/headers/private/shared/Referenceable.h @@ -30,7 +30,7 @@ protected: virtual void LastReferenceReleased(); protected: - vint32 fReferenceCount; + int32 fReferenceCount; }; diff --git a/headers/private/shared/WeakReferenceable.h b/headers/private/shared/WeakReferenceable.h index c09bb0c27d..7d1774fe32 100644 --- a/headers/private/shared/WeakReferenceable.h +++ b/headers/private/shared/WeakReferenceable.h @@ -30,7 +30,7 @@ public: void GetUnchecked(); private: - vint32 fUseCount; + int32 fUseCount; BWeakReferenceable* fObject; }; diff --git a/headers/private/shared/locks.h b/headers/private/shared/locks.h index 052f24b6ec..23b227d2c8 100644 --- a/headers/private/shared/locks.h +++ b/headers/private/shared/locks.h @@ -71,7 +71,7 @@ int32 recursive_lock_get_recursion(recursive_lock *lock); #define INIT_ONCE_UNINITIALIZED -1 #define INIT_ONCE_INITIALIZED -4 -status_t __init_once(vint32* control, status_t (*initRoutine)(void*), +status_t __init_once(int32* control, status_t (*initRoutine)(void*), void* data); #ifdef __cplusplus diff --git a/headers/private/support/StringPrivate.h b/headers/private/support/StringPrivate.h index 799151b66a..0ad274c8c4 100644 --- a/headers/private/support/StringPrivate.h +++ b/headers/private/support/StringPrivate.h @@ -32,12 +32,12 @@ public: return fString._IsShareable(); } - static vint32& DataRefCount(char* data) + static int32& DataRefCount(char* data) { return *(((int32 *)data) - 2); } - vint32& DataRefCount() + int32& DataRefCount() { return DataRefCount(Data()); } diff --git a/src/add-ons/kernel/bus_managers/ps2/ps2_dev.h b/src/add-ons/kernel/bus_managers/ps2/ps2_dev.h index 229dbe0c0f..98c8c45cdb 100644 --- a/src/add-ons/kernel/bus_managers/ps2/ps2_dev.h +++ b/src/add-ons/kernel/bus_managers/ps2/ps2_dev.h @@ -28,7 +28,7 @@ struct ps2_dev { bool active; uint8 idx; sem_id result_sem; - vint32 flags; + int32 flags; uint8 * result_buf; int result_buf_idx; int result_buf_cnt; diff --git a/src/add-ons/kernel/drivers/input/usb_hid/HIDDevice.cpp b/src/add-ons/kernel/drivers/input/usb_hid/HIDDevice.cpp index c1e15e1e3b..9348370891 100644 --- a/src/add-ons/kernel/drivers/input/usb_hid/HIDDevice.cpp +++ b/src/add-ons/kernel/drivers/input/usb_hid/HIDDevice.cpp @@ -264,7 +264,7 @@ HIDDevice::MaybeScheduleTransfer() if (fRemoved) return B_ERROR; - if (atomic_set(&fTransferScheduled, 1) != 0) { + if (atomic_get_and_set(&fTransferScheduled, 1) != 0) { // someone else already caused a transfer to be scheduled return B_OK; } diff --git a/src/add-ons/kernel/drivers/power/acpi_battery/acpi_battery.h b/src/add-ons/kernel/drivers/power/acpi_battery/acpi_battery.h index d712a4fa7b..ffd11dd32b 100644 --- a/src/add-ons/kernel/drivers/power/acpi_battery/acpi_battery.h +++ b/src/add-ons/kernel/drivers/power/acpi_battery/acpi_battery.h @@ -33,7 +33,7 @@ struct battery_driver_cookie { struct battery_device_cookie { battery_driver_cookie* driver_cookie; - vint32 stop_watching; + int32 stop_watching; }; diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp b/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp index b332c49ad3..eaf29c2850 100644 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp +++ b/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp @@ -35,7 +35,7 @@ static device_manager_info *sDeviceManager; static ConditionVariable sFrequencyCondition; -static vint32 sCurrentID; +static int32 sCurrentID; static status_t diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h b/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h index d511f38dd7..b51fa9120f 100644 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h +++ b/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h @@ -22,7 +22,7 @@ struct est_cookie { freq_info* available_states; uint8 number_states; - vint32 stop_watching; + int32 stop_watching; }; diff --git a/src/add-ons/kernel/file_systems/nfs4/FileSystem.h b/src/add-ons/kernel/file_systems/nfs4/FileSystem.h index 5ca172d484..5fdca263a1 100644 --- a/src/add-ons/kernel/file_systems/nfs4/FileSystem.h +++ b/src/add-ons/kernel/file_systems/nfs4/FileSystem.h @@ -109,7 +109,7 @@ private: RPC::Server* fServer; - vint64 fId; + int64 fId; dev_t fDevId; InodeIdMap fInoIdMap; diff --git a/src/add-ons/kernel/network/datalink_protocols/arp/arp.cpp b/src/add-ons/kernel/network/datalink_protocols/arp/arp.cpp index 23a7eaacd1..d2ee6eba0d 100644 --- a/src/add-ons/kernel/network/datalink_protocols/arp/arp.cpp +++ b/src/add-ons/kernel/network/datalink_protocols/arp/arp.cpp @@ -187,7 +187,7 @@ put_request_buffer(arp_entry* entry, net_buffer* buffer) static void delete_request_buffer(arp_entry* entry) { - net_buffer* buffer = atomic_pointer_set(&entry->request_buffer, + net_buffer* buffer = atomic_pointer_get_and_set(&entry->request_buffer, kDeletedBuffer); if (buffer != NULL && buffer != kDeletedBuffer) gBufferModule->free(buffer); diff --git a/src/add-ons/kernel/network/datalink_protocols/ipv6_datagram/ipv6_datagram.cpp b/src/add-ons/kernel/network/datalink_protocols/ipv6_datagram/ipv6_datagram.cpp index e38c43326a..e510e53e56 100644 --- a/src/add-ons/kernel/network/datalink_protocols/ipv6_datagram/ipv6_datagram.cpp +++ b/src/add-ons/kernel/network/datalink_protocols/ipv6_datagram/ipv6_datagram.cpp @@ -251,7 +251,7 @@ put_request_buffer(ndp_entry* entry, net_buffer* buffer) static void delete_request_buffer(ndp_entry* entry) { - net_buffer* buffer = atomic_pointer_set(&entry->request_buffer, + net_buffer* buffer = atomic_pointer_get_and_set(&entry->request_buffer, kDeletedBuffer); if (buffer != NULL && buffer != kDeletedBuffer) gBufferModule->free(buffer); diff --git a/src/add-ons/kernel/network/stack/net_buffer.cpp b/src/add-ons/kernel/network/stack/net_buffer.cpp index d302c6a7e7..8265d92f2b 100644 --- a/src/add-ons/kernel/network/stack/net_buffer.cpp +++ b/src/add-ons/kernel/network/stack/net_buffer.cpp @@ -166,12 +166,12 @@ static status_t read_data(net_buffer* _buffer, size_t offset, void* data, #if ENABLE_STATS -static vint32 sAllocatedDataHeaderCount = 0; -static vint32 sAllocatedNetBufferCount = 0; -static vint32 sEverAllocatedDataHeaderCount = 0; -static vint32 sEverAllocatedNetBufferCount = 0; -static vint32 sMaxAllocatedDataHeaderCount = 0; -static vint32 sMaxAllocatedNetBufferCount = 0; +static int32 sAllocatedDataHeaderCount = 0; +static int32 sAllocatedNetBufferCount = 0; +static int32 sEverAllocatedDataHeaderCount = 0; +static int32 sEverAllocatedNetBufferCount = 0; +static int32 sMaxAllocatedDataHeaderCount = 0; +static int32 sMaxAllocatedNetBufferCount = 0; #endif diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPMailbox.h b/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPMailbox.h index b814919262..b8c53dcd76 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPMailbox.h +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPMailbox.h @@ -90,7 +90,7 @@ private: ExpungeHandler fExpungeHandler; FlagsHandler fFlagsHandler; - vint32 fWatching; + int32 fWatching; BString fSelectedMailbox; diff --git a/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPProtocol.h b/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPProtocol.h index 833e9b7873..bd236fa71a 100644 --- a/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPProtocol.h +++ b/src/add-ons/mail_daemon/inbound_protocols/imap/imap_lib/IMAPProtocol.h @@ -120,7 +120,7 @@ private: BString fCommandError; - vint32 fStopNow; + int32 fStopNow; bool fIsConnected; }; diff --git a/src/apps/activitymonitor/ActivityView.h b/src/apps/activitymonitor/ActivityView.h index 17ac4f89d1..4c9488ec40 100644 --- a/src/apps/activitymonitor/ActivityView.h +++ b/src/apps/activitymonitor/ActivityView.h @@ -76,7 +76,7 @@ public: void RemoveAllDataSources(); bigtime_t RefreshInterval() const - { return atomic_get64((vint64*)&fRefreshInterval); } + { return atomic_get64((int64*)&fRefreshInterval); } protected: virtual void AttachedToWindow(); diff --git a/src/apps/powerstatus/DriverInterface.h b/src/apps/powerstatus/DriverInterface.h index 52194b68e8..fd196bc518 100644 --- a/src/apps/powerstatus/DriverInterface.h +++ b/src/apps/powerstatus/DriverInterface.h @@ -67,7 +67,7 @@ public: protected: virtual void _WatchPowerStatus() = 0; - vint32 fIsWatching; + int32 fIsWatching; sem_id fWaitSem; private: diff --git a/src/build/libroot/atomic.cpp b/src/build/libroot/atomic.cpp index f7a726915a..38aed02692 100644 --- a/src/build/libroot/atomic.cpp +++ b/src/build/libroot/atomic.cpp @@ -6,8 +6,15 @@ #include +void +atomic_set(int32 *value, int32 newValue) +{ + *value = newValue; +} + + int32 -atomic_set(vint32 *value, int32 newValue) +atomic_get_and_set(int32 *value, int32 newValue) { int32 oldValue = *value; *value = newValue; @@ -16,7 +23,7 @@ atomic_set(vint32 *value, int32 newValue) int32 -atomic_test_and_set(vint32 *value, int32 newValue, int32 testAgainst) +atomic_test_and_set(int32 *value, int32 newValue, int32 testAgainst) { int32 oldValue = *value; if (oldValue == testAgainst) @@ -26,7 +33,7 @@ atomic_test_and_set(vint32 *value, int32 newValue, int32 testAgainst) int32 -atomic_add(vint32 *value, int32 addValue) +atomic_add(int32 *value, int32 addValue) { int32 oldValue = *value; *value += addValue; @@ -35,7 +42,7 @@ atomic_add(vint32 *value, int32 addValue) int32 -atomic_and(vint32 *value, int32 andValue) +atomic_and(int32 *value, int32 andValue) { int32 oldValue = *value; *value &= andValue; @@ -44,7 +51,7 @@ atomic_and(vint32 *value, int32 andValue) int32 -atomic_or(vint32 *value, int32 orValue) +atomic_or(int32 *value, int32 orValue) { int32 oldValue = *value; *value |= orValue; @@ -53,14 +60,21 @@ atomic_or(vint32 *value, int32 orValue) int32 -atomic_get(vint32 *value) +atomic_get(int32 *value) { return *value; } +void +atomic_set64(int64 *value, int64 newValue) +{ + *value = newValue; +} + + int64 -atomic_set64(vint64 *value, int64 newValue) +atomic_get_and_set64(int64 *value, int64 newValue) { int64 oldValue = *value; *value = newValue; @@ -68,7 +82,7 @@ atomic_set64(vint64 *value, int64 newValue) } int64 -atomic_test_and_set64(vint64 *value, int64 newValue, int64 testAgainst) +atomic_test_and_set64(int64 *value, int64 newValue, int64 testAgainst) { int64 oldValue = *value; if (oldValue == testAgainst) @@ -77,7 +91,7 @@ atomic_test_and_set64(vint64 *value, int64 newValue, int64 testAgainst) } int64 -atomic_add64(vint64 *value, int64 addValue) +atomic_add64(int64 *value, int64 addValue) { int64 oldValue = *value; *value += addValue; @@ -85,7 +99,7 @@ atomic_add64(vint64 *value, int64 addValue) } int64 -atomic_and64(vint64 *value, int64 andValue) +atomic_and64(int64 *value, int64 andValue) { int64 oldValue = *value; *value &= andValue; @@ -93,7 +107,7 @@ atomic_and64(vint64 *value, int64 andValue) } int64 -atomic_or64(vint64 *value, int64 orValue) +atomic_or64(int64 *value, int64 orValue) { int64 oldValue = *value; *value |= orValue; @@ -101,7 +115,7 @@ atomic_or64(vint64 *value, int64 orValue) } int64 -atomic_get64(vint64 *value) +atomic_get64(int64 *value) { return *value; } diff --git a/src/kits/support/String.cpp b/src/kits/support/String.cpp index 56b6084e93..b5cfaafef4 100644 --- a/src/kits/support/String.cpp +++ b/src/kits/support/String.cpp @@ -183,14 +183,14 @@ BStringRef::operator&() // #pragma mark - BString -inline vint32& +inline int32& BString::_ReferenceCount() { return Private::DataRefCount(fPrivateData); } -inline const vint32& +inline const int32& BString::_ReferenceCount() const { return Private::DataRefCount(fPrivateData); diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp index 097ea2b07c..9e460a572c 100644 --- a/src/system/kernel/UserEvent.cpp +++ b/src/system/kernel/UserEvent.cpp @@ -33,7 +33,7 @@ struct SignalEvent::EventSignal : Signal { bool MarkUsed() { - return atomic_set(&fInUse, 1) != 0; + return atomic_get_and_set(&fInUse, 1) != 0; } void SetUnused() @@ -216,7 +216,7 @@ CreateThreadEvent::Create(const ThreadCreationAttributes& attributes) status_t CreateThreadEvent::Fire() { - bool wasPending = atomic_set(&fPendingDPC, 1) != 0; + bool wasPending = atomic_get_and_set(&fPendingDPC, 1) != 0; if (wasPending) return B_BUSY; diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 91cc6bf31a..be1afdae0a 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -205,7 +205,7 @@ set_mtrr(void* _parameter, int cpu) // sCpuRendezvous2 before the last CPU has actually left the loop in // smp_cpu_rendezvous(); if (cpu == 0) - atomic_set((vint32*)&sCpuRendezvous3, 0); + atomic_set((int32*)&sCpuRendezvous3, 0); disable_caches(); @@ -233,7 +233,7 @@ set_mtrrs(void* _parameter, int cpu) // sCpuRendezvous2 before the last CPU has actually left the loop in // smp_cpu_rendezvous(); if (cpu == 0) - atomic_set((vint32*)&sCpuRendezvous3, 0); + atomic_set((int32*)&sCpuRendezvous3, 0); disable_caches(); @@ -259,7 +259,7 @@ init_mtrrs(void* _unused, int cpu) // sCpuRendezvous2 before the last CPU has actually left the loop in // smp_cpu_rendezvous(); if (cpu == 0) - atomic_set((vint32*)&sCpuRendezvous3, 0); + atomic_set((int32*)&sCpuRendezvous3, 0); disable_caches(); diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h index f781f7df1f..8df3181552 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h @@ -108,7 +108,7 @@ X86PagingMethod32Bit::Method() X86PagingMethod32Bit::SetPageTableEntry(page_table_entry* entry, page_table_entry newEntry) { - return atomic_set((int32*)entry, newEntry); + return atomic_get_and_set((int32*)entry, newEntry); } diff --git a/src/system/kernel/arch/x86/paging/64bit/X86PagingMethod64Bit.h b/src/system/kernel/arch/x86/paging/64bit/X86PagingMethod64Bit.h index e834434c78..ac1bfebbc4 100644 --- a/src/system/kernel/arch/x86/paging/64bit/X86PagingMethod64Bit.h +++ b/src/system/kernel/arch/x86/paging/64bit/X86PagingMethod64Bit.h @@ -116,7 +116,7 @@ X86PagingMethod64Bit::Method() /*static*/ inline uint64 X86PagingMethod64Bit::SetTableEntry(uint64* entry, uint64 newEntry) { - return atomic_set64((int64*)entry, newEntry); + return atomic_get_and_set64((int64*)entry, newEntry); } diff --git a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h index 6d6b1ed1ab..c787bdc0cb 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h +++ b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h @@ -158,7 +158,7 @@ X86PagingMethodPAE::PageDirEntryForAddress( X86PagingMethodPAE::SetPageTableEntry(pae_page_table_entry* entry, pae_page_table_entry newEntry) { - return atomic_set64((int64*)entry, newEntry); + return atomic_get_and_set64((int64*)entry, newEntry); } diff --git a/src/system/kernel/fs/EntryCache.cpp b/src/system/kernel/fs/EntryCache.cpp index 2c44d60e88..e9a6a14db9 100644 --- a/src/system/kernel/fs/EntryCache.cpp +++ b/src/system/kernel/fs/EntryCache.cpp @@ -165,7 +165,8 @@ EntryCache::Lookup(ino_t dirID, const char* name, ino_t& _nodeID) if (entry == NULL) return false; - int32 oldGeneration = atomic_set(&entry->generation, fCurrentGeneration); + int32 oldGeneration = atomic_get_and_set(&entry->generation, + fCurrentGeneration); if (oldGeneration == fCurrentGeneration || entry->index < 0) { // The entry is already in the current generation or is being moved to // it by another thread. diff --git a/src/system/kernel/fs/EntryCache.h b/src/system/kernel/fs/EntryCache.h index 91d74043ed..e197426eec 100644 --- a/src/system/kernel/fs/EntryCache.h +++ b/src/system/kernel/fs/EntryCache.h @@ -34,14 +34,14 @@ struct EntryCacheEntry { EntryCacheEntry* hash_link; ino_t node_id; ino_t dir_id; - vint32 generation; - vint32 index; + int32 generation; + int32 index; char name[1]; }; struct EntryCacheGeneration { - vint32 next_index; + int32 next_index; EntryCacheEntry** entries; EntryCacheGeneration(); diff --git a/src/system/kernel/fs/unused_vnodes.h b/src/system/kernel/fs/unused_vnodes.h index ca866debab..5f5bb490b6 100644 --- a/src/system/kernel/fs/unused_vnodes.h +++ b/src/system/kernel/fs/unused_vnodes.h @@ -30,15 +30,15 @@ const static uint32 kMaxUnusedVnodes = 8192; */ static mutex sUnusedVnodesLock = MUTEX_INITIALIZER("unused vnodes"); static list sUnusedVnodeList; -static vuint32 sUnusedVnodes = 0; +static uint32 sUnusedVnodes = 0; static const int32 kMaxHotVnodes = 1024; static rw_lock sHotVnodesLock = RW_LOCK_INITIALIZER("hot vnodes"); static Vnode* sHotVnodes[kMaxHotVnodes]; -static vint32 sNextHotVnodeIndex = 0; +static int32 sNextHotVnodeIndex = 0; static const int32 kUnusedVnodesCheckInterval = 64; -static vint32 sUnusedVnodesCheckCount = 0; +static int32 sUnusedVnodesCheckCount = 0; /*! Must be called with sHotVnodesLock write-locked. @@ -48,7 +48,7 @@ flush_hot_vnodes_locked() { MutexLocker unusedLocker(sUnusedVnodesLock); - int32 count = std::min((int32)sNextHotVnodeIndex, kMaxHotVnodes); + int32 count = std::min(sNextHotVnodeIndex, kMaxHotVnodes); for (int32 i = 0; i < count; i++) { Vnode* vnode = sHotVnodes[i]; if (vnode == NULL) @@ -87,7 +87,7 @@ vnode_unused(Vnode* vnode) bool result = false; int32 checkCount = atomic_add(&sUnusedVnodesCheckCount, 1); if (checkCount == kUnusedVnodesCheckInterval) { - uint32 unusedCount = sUnusedVnodes; + uint32 unusedCount = atomic_get((int32*)&sUnusedVnodes); if (unusedCount > kMaxUnusedVnodes && low_resource_state( B_KERNEL_RESOURCE_PAGES | B_KERNEL_RESOURCE_MEMORY) @@ -164,7 +164,7 @@ vnode_to_be_freed(Vnode* vnode) if (vnode->IsHot()) { // node is hot -- remove it from the array // TODO: Maybe better completely flush the array while at it? - int32 count = sNextHotVnodeIndex; + int32 count = atomic_get(&sNextHotVnodeIndex); count = std::min(count, kMaxHotVnodes); for (int32 i = 0; i < count; i++) { if (sHotVnodes[i] == vnode) { diff --git a/src/system/kernel/posix/xsi_message_queue.cpp b/src/system/kernel/posix/xsi_message_queue.cpp index f0141b1e93..247a1aeda1 100644 --- a/src/system/kernel/posix/xsi_message_queue.cpp +++ b/src/system/kernel/posix/xsi_message_queue.cpp @@ -385,8 +385,8 @@ static mutex sIpcLock; static mutex sXsiMessageQueueLock; static uint32 sGlobalSequenceNumber = 1; -static vint32 sXsiMessageCount = 0; -static vint32 sXsiMessageQueueCount = 0; +static int32 sXsiMessageCount = 0; +static int32 sXsiMessageQueueCount = 0; // #pragma mark - @@ -690,7 +690,7 @@ _user_xsi_msgget(key_t key, int flags) if (create) { // Create a new message queue for this key - if (sXsiMessageQueueCount >= MAX_XSI_MESSAGE_QUEUE) { + if (atomic_get(&sXsiMessageQueueCount) >= MAX_XSI_MESSAGE_QUEUE) { TRACE_ERROR(("xsi_msgget: reached limit of maximun number of " "message queues\n")); return ENOSPC; diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 93be92fe2e..1b2f8867ff 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -79,9 +79,9 @@ enum mailbox_source { MAILBOX_BCAST, }; -static vint32 sBootCPUSpin = 0; +static int32 sBootCPUSpin = 0; -static vint32 sEarlyCPUCall = 0; +static int32 sEarlyCPUCall = 0; static void (*sEarlyCPUCallFunction)(void*, int); void* sEarlyCPUCallCookie; @@ -109,7 +109,7 @@ static struct { spinlock *lock; } sLastCaller[NUM_LAST_CALLERS]; -static vint32 sLastIndex = 0; +static int32 sLastIndex = 0; // Is incremented atomically. Must be % NUM_LAST_CALLERS before being used // as index into sLastCaller. Note, that it has to be casted to uint32 // before applying the modulo operation, since otherwise after overflowing @@ -129,7 +129,7 @@ push_lock_caller(void* caller, spinlock* lock) static void* find_lock_caller(spinlock* lock) { - int32 lastIndex = (uint32)sLastIndex % NUM_LAST_CALLERS; + int32 lastIndex = (uint32)atomic_get(&sLastIndex) % NUM_LAST_CALLERS; for (int32 i = 0; i < NUM_LAST_CALLERS; i++) { int32 index = (NUM_LAST_CALLERS + lastIndex - 1 - i) % NUM_LAST_CALLERS; @@ -522,7 +522,7 @@ bool try_acquire_write_seqlock(seqlock* lock) { bool succeed = try_acquire_spinlock(&lock->lock); if (succeed) - atomic_add(&lock->count, 1); + atomic_add((int32*)&lock->count, 1); return succeed; } @@ -530,26 +530,26 @@ try_acquire_write_seqlock(seqlock* lock) { void acquire_write_seqlock(seqlock* lock) { acquire_spinlock(&lock->lock); - atomic_add(&lock->count, 1); + atomic_add((int32*)&lock->count, 1); } void release_write_seqlock(seqlock* lock) { - atomic_add(&lock->count, 1); + atomic_add((int32*)&lock->count, 1); release_spinlock(&lock->lock); } uint32 acquire_read_seqlock(seqlock* lock) { - return atomic_get(&lock->count); + return atomic_get((int32*)&lock->count); } bool release_read_seqlock(seqlock* lock, uint32 count) { - uint32 current = atomic_get(&lock->count); + uint32 current = atomic_get((int32*)&lock->count); if (count % 2 == 1 || current != count) { PAUSE(); @@ -870,10 +870,10 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie) uint32 cpuMask = (1 << sNumCPUs) - 2; // all CPUs but the boot cpu - sEarlyCPUCall = cpuMask; + atomic_set(&sEarlyCPUCall, cpuMask); // wait for all CPUs to finish - while ((sEarlyCPUCall & cpuMask) != 0) + while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0) PAUSE(); } @@ -1166,8 +1166,8 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous) smp_cpu_rendezvous(rendezVous, cpu); - while (sBootCPUSpin == 0) { - if ((sEarlyCPUCall & (1 << cpu)) != 0) + while (atomic_get(&sBootCPUSpin) == 0) { + if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0) process_early_cpu_call(cpu); PAUSE(); @@ -1185,7 +1185,7 @@ smp_wake_up_non_boot_cpus() sICIEnabled = true; // resume non boot CPUs - sBootCPUSpin = 1; + atomic_set(&sBootCPUSpin, 1); } @@ -1200,11 +1200,12 @@ smp_wake_up_non_boot_cpus() ensured via another rendez-vous) the variable can be reset. */ void -smp_cpu_rendezvous(volatile uint32* var, int current_cpu) +smp_cpu_rendezvous(uint32* var, int current_cpu) { - atomic_or((vint32*)var, 1 << current_cpu); + atomic_or((int32*)var, 1 << current_cpu); - while (*var != (((uint32)1 << sNumCPUs) - 1)) + uint32 allReady = ((uint32)1 << sNumCPUs) - 1; + while ((uint32)atomic_get((int32*)var) != allReady) PAUSE(); } diff --git a/src/system/libroot/os/arch/x86/atomic.S b/src/system/libroot/os/arch/x86/atomic.S index 31b46f1b06..9e26dfde79 100644 --- a/src/system/libroot/os/arch/x86/atomic.S +++ b/src/system/libroot/os/arch/x86/atomic.S @@ -1,4 +1,7 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + * ** Copyright 2003, Marcus Overhagen. All rights reserved. ** Distributed under the terms of the OpenBeOS license. ** @@ -6,96 +9,133 @@ ** Distributed under the terms of the NewOS License. */ + #include .text -/* int32 atomic_set(vint32 *value, int32 newValue) */ + +/* void atomic_set(int32* value, int32 newValue) */ FUNCTION(atomic_set): - movl 4(%esp),%edx - movl 8(%esp),%eax + movl 4(%esp), %edx + movl 8(%esp), %eax lock - xchg %eax,(%edx) + addl $0, (%esp) + movl %eax, (%edx) ret FUNCTION_END(atomic_set) -/* int32 atomic_test_and_set(vint32 *value, int32 newValue, int32 testAgainst) */ + +/* int32 atomic_get_and_set(int32* value, int32 newValue) */ +FUNCTION(atomic_get_and_set): + movl 4(%esp), %edx + movl 8(%esp), %eax + xchg %eax, (%edx) + ret +FUNCTION_END(atomic_get_and_set) + + +/* int32 atomic_test_and_set(int32* value, int32 newValue, + int32 testAgainst) */ FUNCTION(atomic_test_and_set): - movl 4(%esp),%edx - movl 8(%esp),%ecx - movl 12(%esp),%eax + movl 4(%esp), %edx + movl 8(%esp), %ecx + movl 12(%esp), %eax lock - cmpxchgl %ecx,(%edx) + cmpxchgl %ecx, (%edx) ret FUNCTION_END(atomic_test_and_set) -/* int32 atomic_add(vint32 *value, int32 addValue) */ + +/* int32 atomic_add(int32* value, int32 addValue) */ FUNCTION(atomic_add): - movl 4(%esp),%edx - movl 8(%esp),%eax + movl 4(%esp), %edx + movl 8(%esp), %eax lock - xaddl %eax,(%edx) + xaddl %eax, (%edx) ret FUNCTION_END(atomic_add) -/* int32 atomic_and(vint32 *value, int32 andValue) */ + +/* int32 atomic_and(int32* value, int32 andValue) */ FUNCTION(atomic_and): - movl 4(%esp),%edx -_atomic_and1: - movl 8(%esp),%ecx - movl (%edx),%eax - andl %eax,%ecx + movl 4(%esp), %edx +1: + movl 8(%esp), %ecx + movl (%edx), %eax + andl %eax, %ecx lock - cmpxchgl %ecx,(%edx) - jnz _atomic_and1 + cmpxchgl %ecx, (%edx) + jnz 1b ret FUNCTION_END(atomic_and) -/* int32 atomic_or(vint32 *value, int32 orValue) */ + +/* int32 atomic_or(int32* value, int32 orValue) */ FUNCTION(atomic_or): - movl 4(%esp),%edx -_atomic_or1: - movl 8(%esp),%ecx - movl (%edx),%eax - orl %eax,%ecx + movl 4(%esp), %edx +1: + movl 8(%esp), %ecx + movl (%edx), %eax + orl %eax, %ecx lock - cmpxchgl %ecx,(%edx) - jnz _atomic_or1 + cmpxchgl %ecx, (%edx) + jnz 1b ret FUNCTION_END(atomic_or) -/* int32 atomic_get(vint32 *value) */ + +/* int32 atomic_get(int32* value) */ FUNCTION(atomic_get): movl 4(%esp), %edx -_atomic_get1: movl (%edx), %eax - movl %eax, %ecx lock - cmpxchgl %ecx, (%edx) - jnz _atomic_get1 + addl $0, (%esp) ret FUNCTION_END(atomic_get) -/* int64 atomic_set64(vint64 *value, int64 newValue) */ + +/* void atomic_set64(int64* value, int64 newValue) */ FUNCTION(atomic_set64): push %esi push %ebx movl 12(%esp), %esi /* value */ movl 16(%esp), %ebx /* newValue low */ movl 20(%esp), %ecx /* newValue high */ -_atomic_set64_1: +1: movl (%esi), %eax /* testAgainst low */ movl 4(%esi), %edx /* testAgainst high */ lock cmpxchg8b (%esi) - jnz _atomic_set64_1 + jnz 1b pop %ebx pop %esi ret FUNCTION_END(atomic_set64) -/* int64 atomic_test_and_set64(vint64 *value, int64 newValue, int64 testAgainst) */ + +/* void atomic_get_and_set64(int64* value, int64 newValue) */ +FUNCTION(atomic_get_and_set64): + push %esi + push %ebx + movl 12(%esp), %esi /* value */ + movl 16(%esp), %ebx /* newValue low */ + movl 20(%esp), %ecx /* newValue high */ +1: + movl (%esi), %eax /* testAgainst low */ + movl 4(%esi), %edx /* testAgainst high */ + lock + cmpxchg8b (%esi) + jnz 1b + pop %ebx + pop %esi + ret +FUNCTION_END(atomic_get_and_set64) + + +/* int64 atomic_test_and_set64(int64* value, int64 newValue, + int64 testAgainst) */ FUNCTION(atomic_test_and_set64): push %esi push %ebx @@ -111,12 +151,13 @@ FUNCTION(atomic_test_and_set64): ret FUNCTION_END(atomic_test_and_set64) -/* int64 atomic_add64(vint64 *value, int64 addValue) */ + +/* int64 atomic_add64(int64* value, int64 addValue) */ FUNCTION(atomic_add64): push %esi push %ebx movl 12(%esp), %esi -_atomic_add64_1: +1: movl (%esi), %eax movl 4(%esi), %edx movl %eax, %ebx @@ -125,18 +166,18 @@ _atomic_add64_1: adcl 20(%esp), %ecx lock cmpxchg8b (%esi) - jnz _atomic_add64_1 + jnz 1b pop %ebx pop %esi ret FUNCTION_END(atomic_add64) -/* int64 atomic_and64(vint64 *value, int64 andValue) */ +/* int64 atomic_and64(int64* value, int64 andValue) */ FUNCTION(atomic_and64): push %esi push %ebx movl 12(%esp), %esi -_atomic_and64_1: +1: movl (%esi), %eax movl 4(%esi), %edx movl %eax, %ebx @@ -145,18 +186,19 @@ _atomic_and64_1: andl 20(%esp), %ecx lock cmpxchg8b (%esi) - jnz _atomic_and64_1 + jnz 1b pop %ebx pop %esi ret FUNCTION_END(atomic_and64) -/* int64 atomic_or64(vint64 *value, int64 orValue) */ + +/* int64 atomic_or64(int64* value, int64 orValue) */ FUNCTION(atomic_or64): push %esi push %ebx movl 12(%esp), %esi -_atomic_or64_1: +1: movl (%esi), %eax movl 4(%esi), %edx movl %eax, %ebx @@ -165,26 +207,28 @@ _atomic_or64_1: orl 20(%esp), %ecx lock cmpxchg8b (%esi) - jnz _atomic_or64_1 + jnz 1b pop %ebx pop %esi ret FUNCTION_END(atomic_or64) -/* int64 atomic_get64(vint64 *value) */ + +/* int64 atomic_get64(int64* value) */ FUNCTION(atomic_get64): push %esi push %ebx movl 12(%esp), %esi -_atomic_get64_1: +1: movl (%esi), %eax movl 4(%esi), %edx movl %eax, %ebx movl %edx, %ecx lock cmpxchg8b (%esi) - jnz _atomic_get64_1 + jnz 1b pop %ebx pop %esi ret FUNCTION_END(atomic_get64) + diff --git a/src/system/libroot/os/arch/x86_64/atomic.S b/src/system/libroot/os/arch/x86_64/atomic.S index 7abf2d2743..5c905ceb01 100644 --- a/src/system/libroot/os/arch/x86_64/atomic.S +++ b/src/system/libroot/os/arch/x86_64/atomic.S @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. * Distributed under the terms of the MIT License. */ @@ -9,15 +10,23 @@ .text -/* int32 atomic_set(vint32 *value, int32 newValue) */ + +/* int32 atomic_set(int32* value, int32 newValue) */ FUNCTION(atomic_set): - movl %esi, %eax - lock - xchgl %eax, (%rdi) + sfence + movl %esi, (%rdi) ret FUNCTION_END(atomic_set) -/* int32 atomic_test_and_set(vint32 *value, int32 newValue, int32 testAgainst) */ + +/* int32 atomic_get_and_set(int32* value, int32 newValue) */ +FUNCTION(atomic_get_and_set): + xchgl %esi, (%rdi) + ret +FUNCTION_END(atomic_get_and_set) + + +/* int32 atomic_test_and_set(int32* value, int32 newValue, int32 testAgainst) */ FUNCTION(atomic_test_and_set): movl %edx, %eax lock @@ -25,15 +34,16 @@ FUNCTION(atomic_test_and_set): ret FUNCTION_END(atomic_test_and_set) -/* int32 atomic_add(vint32 *value, int32 addValue) */ + +/* int32 atomic_add(int32* value, int32 addValue) */ FUNCTION(atomic_add): - movl %esi, %eax lock - xaddl %eax, (%rdi) + xaddl %esi, (%rdi) ret FUNCTION_END(atomic_add) -/* int32 atomic_and(vint32 *value, int32 andValue) */ + +/* int32 atomic_and(int32* value, int32 andValue) */ FUNCTION(atomic_and): movl (%rdi), %eax 1: movl %eax, %edx @@ -46,7 +56,8 @@ FUNCTION(atomic_and): ret FUNCTION_END(atomic_and) -/* int32 atomic_or(vint32 *value, int32 orValue) */ + +/* int32 atomic_or(int32* value, int32 orValue) */ FUNCTION(atomic_or): movl (%rdi), %eax 1: movl %eax, %edx @@ -59,24 +70,31 @@ FUNCTION(atomic_or): ret FUNCTION_END(atomic_or) -/* int32 atomic_get(vint32 *value) */ + +/* int32 atomic_get(int32* value) */ FUNCTION(atomic_get): movl (%rdi), %eax -1: lock - cmpxchgl %eax, (%rdi) - jnz 1b - ret + lfence FUNCTION_END(atomic_get) -/* int64 atomic_set64(vint64 *value, int64 newValue) */ + +/* int64 atomic_set64(int64* value, int64 newValue) */ FUNCTION(atomic_set64): - movq %rsi, %rax - lock - xchgq %rax, (%rdi) + sfence + movq %rsi, (%rdi) ret FUNCTION_END(atomic_set64) -/* int64 atomic_test_and_set64(vint64 *value, int64 newValue, int64 testAgainst) */ + +/* int64 atomic_get_and_set64(int64* value, int64 newValue) */ +FUNCTION(atomic_get_and_set64): + xchgq %rsi, (%rdi) + ret +FUNCTION_END(atomic_get_and_set64) + + +/* int64 atomic_test_and_set64(int64* value, int64 newValue, + int64 testAgainst) */ FUNCTION(atomic_test_and_set64): movq %rdx, %rax lock @@ -84,7 +102,8 @@ FUNCTION(atomic_test_and_set64): ret FUNCTION_END(atomic_test_and_set64) -/* int64 atomic_add64(vint64 *value, int64 addValue) */ + +/* int64 atomic_add64(int64* value, int64 addValue) */ FUNCTION(atomic_add64): movq %rsi, %rax lock @@ -92,7 +111,8 @@ FUNCTION(atomic_add64): ret FUNCTION_END(atomic_add64) -/* int64 atomic_and64(vint64 *value, int64 andValue) */ + +/* int64 atomic_and64(int64* value, int64 andValue) */ FUNCTION(atomic_and64): movq (%rdi), %rax 1: movq %rax, %rdx @@ -105,7 +125,8 @@ FUNCTION(atomic_and64): ret FUNCTION_END(atomic_and64) -/* int64 atomic_or64(vint64 *value, int64 orValue) */ + +/* int64 atomic_or64(int64* value, int64 orValue) */ FUNCTION(atomic_or64): movq (%rdi), %rax 1: movq %rax, %rdx @@ -118,11 +139,10 @@ FUNCTION(atomic_or64): ret FUNCTION_END(atomic_or64) -/* int64 atomic_get64(vint64 *value) */ + +/* int64 atomic_get64(int64* value) */ FUNCTION(atomic_get64): movq (%rdi), %rax -1: lock - cmpxchgq %rax, (%rdi) - jnz 1b - ret + lfence FUNCTION_END(atomic_get64) + diff --git a/src/system/libroot/os/locks/init_once.cpp b/src/system/libroot/os/locks/init_once.cpp index 2ec5e7e824..ce3ad18041 100644 --- a/src/system/libroot/os/locks/init_once.cpp +++ b/src/system/libroot/os/locks/init_once.cpp @@ -16,7 +16,7 @@ enum { status_t -__init_once(vint32* control, status_t (*initRoutine)(void*), void* data) +__init_once(int32* control, status_t (*initRoutine)(void*), void* data) { // Algorithm: // The control variable goes through at most four states: @@ -41,7 +41,7 @@ __init_once(vint32* control, status_t (*initRoutine)(void*), void* data) // we're the first -- perform the initialization initRoutine(data); - value = atomic_set(control, STATE_INITIALIZED); + value = atomic_get_and_set(control, STATE_INITIALIZED); // If someone else is waiting, we need to delete the semaphore. if (value >= 0) diff --git a/src/system/libroot/posix/pthread/pthread_key.cpp b/src/system/libroot/posix/pthread/pthread_key.cpp index 956c98018b..431a4d4481 100644 --- a/src/system/libroot/posix/pthread/pthread_key.cpp +++ b/src/system/libroot/posix/pthread/pthread_key.cpp @@ -106,7 +106,7 @@ pthread_key_delete(pthread_key_t key) if (key < 0 || key >= PTHREAD_KEYS_MAX) return EINVAL; - int32 sequence = atomic_set(&sKeyTable[key].sequence, + int32 sequence = atomic_get_and_set(&sKeyTable[key].sequence, PTHREAD_UNUSED_SEQUENCE); if (sequence == PTHREAD_UNUSED_SEQUENCE) return EINVAL; diff --git a/src/system/libroot/posix/pthread/pthread_once.cpp b/src/system/libroot/posix/pthread/pthread_once.cpp index e83627a571..72926b8fc9 100644 --- a/src/system/libroot/posix/pthread/pthread_once.cpp +++ b/src/system/libroot/posix/pthread/pthread_once.cpp @@ -26,7 +26,8 @@ init_function_canceled(void* data) pthread_once_t* onceControl = (pthread_once_t*)data; // reset the control state to uninitialized - int32 value = atomic_set((vint32*)&onceControl->state, STATE_UNINITIALIZED); + int32 value = atomic_get_and_set((int32*)&onceControl->state, + STATE_UNINITIALIZED); // If someone has set a semaphore, delete it. if (value >= 0) @@ -66,7 +67,8 @@ pthread_once(pthread_once_t* onceControl, void (*initRoutine)(void)) initRoutine(); pthread_cleanup_pop(false); - value = atomic_set((vint32*)&onceControl->state, STATE_INITIALIZED); + value = atomic_get_and_set((int32*)&onceControl->state, + STATE_INITIALIZED); // If someone else is waiting, we need to delete the semaphore. if (value >= 0) @@ -105,7 +107,7 @@ pthread_once(pthread_once_t* onceControl, void (*initRoutine)(void)) return 0; } else if (value == STATE_SPINNING) { // out of semaphores -- spin - while (atomic_get((vint32*)&onceControl->state) == STATE_SPINNING); + while (atomic_get((int32*)&onceControl->state) == STATE_SPINNING); } } } diff --git a/src/tools/fs_shell/atomic.cpp b/src/tools/fs_shell/atomic.cpp index 1899a9fada..ebfcdc56d4 100644 --- a/src/tools/fs_shell/atomic.cpp +++ b/src/tools/fs_shell/atomic.cpp @@ -10,86 +10,100 @@ #include "fssh_atomic.h" -int32_t -fssh_atomic_set(vint32_t *value, int32_t newValue) +void +fssh_atomic_set(int32_t* value, int32_t newValue) { - return atomic_set((vint32*)value, newValue); + atomic_set((int32*)value, newValue); } int32_t -fssh_atomic_test_and_set(vint32_t *value, int32_t newValue, int32_t testAgainst) +fssh_atomic_get_and_set(int32_t* value, int32_t newValue) { - return atomic_test_and_set((vint32*)value, newValue, testAgainst); + return atomic_get_and_set((int32*)value, newValue); } int32_t -fssh_atomic_add(vint32_t *value, int32_t addValue) +fssh_atomic_test_and_set(int32_t *value, int32_t newValue, int32_t testAgainst) { - return atomic_add((vint32*)value, addValue); + return atomic_test_and_set((int32*)value, newValue, testAgainst); } int32_t -fssh_atomic_and(vint32_t *value, int32_t andValue) +fssh_atomic_add(int32_t *value, int32_t addValue) { - return atomic_and((vint32*)value, andValue); + return atomic_add((int32*)value, addValue); } int32_t -fssh_atomic_or(vint32_t *value, int32_t orValue) +fssh_atomic_and(int32_t *value, int32_t andValue) { - return atomic_or((vint32*)value, orValue); + return atomic_and((int32*)value, andValue); } int32_t -fssh_atomic_get(vint32_t *value) +fssh_atomic_or(int32_t *value, int32_t orValue) { - return atomic_get((vint32*)value); + return atomic_or((int32*)value, orValue); +} + + +int32_t +fssh_atomic_get(int32_t *value) +{ + return atomic_get((int32*)value); +} + + +void +fssh_atomic_set64(int64_t *value, int64_t newValue) +{ + atomic_set64((int64*)value, newValue); } int64_t -fssh_atomic_set64(vint64_t *value, int64_t newValue) +fssh_atomic_get_and_set64(int64_t* value, int64_t newValue) { - return atomic_set64((vint64*)value, newValue); + return atomic_get_and_set64((int64*)value, newValue); } int64_t -fssh_atomic_test_and_set64(vint64_t *value, int64_t newValue, int64_t testAgainst) +fssh_atomic_test_and_set64(int64_t *value, int64_t newValue, int64_t testAgainst) { - return atomic_test_and_set64((vint64 *)value, newValue, testAgainst); + return atomic_test_and_set64((int64 *)value, newValue, testAgainst); } int64_t -fssh_atomic_add64(vint64_t *value, int64_t addValue) +fssh_atomic_add64(int64_t *value, int64_t addValue) { - return atomic_add64((vint64*)value, addValue); + return atomic_add64((int64*)value, addValue); } int64_t -fssh_atomic_and64(vint64_t *value, int64_t andValue) +fssh_atomic_and64(int64_t *value, int64_t andValue) { - return atomic_and64((vint64*)value, andValue); + return atomic_and64((int64*)value, andValue); } int64_t -fssh_atomic_or64(vint64_t *value, int64_t orValue) +fssh_atomic_or64(int64_t *value, int64_t orValue) { - return atomic_or64((vint64*)value, orValue); + return atomic_or64((int64*)value, orValue); } int64_t -fssh_atomic_get64(vint64_t *value) +fssh_atomic_get64(int64_t *value) { - return atomic_get64((vint64*)value); + return atomic_get64((int64*)value); } diff --git a/src/tools/fs_shell/vfs.cpp b/src/tools/fs_shell/vfs.cpp index 6a06804d4c..f007068766 100644 --- a/src/tools/fs_shell/vfs.cpp +++ b/src/tools/fs_shell/vfs.cpp @@ -3609,10 +3609,10 @@ common_lock_node(int fd, bool kernel) // We need to set the locking atomically - someone // else might set one at the same time #ifdef __x86_64__ - if (fssh_atomic_test_and_set64((vint64_t *)&vnode->mandatory_locked_by, + if (fssh_atomic_test_and_set64((int64_t *)&vnode->mandatory_locked_by, (fssh_addr_t)descriptor, 0) != 0) #else - if (fssh_atomic_test_and_set((vint32_t *)&vnode->mandatory_locked_by, + if (fssh_atomic_test_and_set((int32_t *)&vnode->mandatory_locked_by, (fssh_addr_t)descriptor, 0) != 0) #endif status = FSSH_B_BUSY; @@ -3637,10 +3637,10 @@ common_unlock_node(int fd, bool kernel) // We need to set the locking atomically - someone // else might set one at the same time #ifdef __x86_64__ - if (fssh_atomic_test_and_set64((vint64_t *)&vnode->mandatory_locked_by, + if (fssh_atomic_test_and_set64((int64_t *)&vnode->mandatory_locked_by, 0, (fssh_addr_t)descriptor) != (int64_t)descriptor) #else - if (fssh_atomic_test_and_set((vint32_t *)&vnode->mandatory_locked_by, + if (fssh_atomic_test_and_set((int32_t *)&vnode->mandatory_locked_by, 0, (fssh_addr_t)descriptor) != (int32_t)descriptor) #endif status = FSSH_B_BAD_VALUE; From 273f2f38cd4b219ac8197888962d0710c149d606 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 5 Nov 2013 22:47:18 +0100 Subject: [PATCH 098/273] kernel: Improve spinlock implementation atomic_or() and atomic_and() are not supported by x86 are need to be emulated using CAS. Use atomic_get_and_set() and atomic_set() instead. --- headers/private/kernel/smp.h | 4 ++-- src/system/kernel/smp.cpp | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index d20ee28e0a..c7aa163762 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -76,7 +76,7 @@ int smp_intercpu_int_handler(int32 cpu); static inline bool try_acquire_spinlock_inline(spinlock* lock) { - return atomic_or((int32*)lock, 1) == 0; + return atomic_get_and_set((int32*)lock, 1) == 0; } @@ -92,7 +92,7 @@ acquire_spinlock_inline(spinlock* lock) static inline void release_spinlock_inline(spinlock* lock) { - atomic_and((int32*)lock, 0); + atomic_set((int32*)lock, 0); } diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 1b2f8867ff..caf29bfe9a 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -313,7 +313,7 @@ try_acquire_spinlock(spinlock* lock) if (atomic_add(&lock->lock, 1) != 0) return false; #else - if (atomic_or((int32*)lock, 1) != 0) + if (atomic_get_and_set((int32*)lock, 1) != 0) return false; # if DEBUG_SPINLOCKS @@ -353,7 +353,7 @@ acquire_spinlock(spinlock* lock) process_all_pending_ici(currentCPU); PAUSE(); } - if (atomic_or((int32*)lock, 1) == 0) + if (atomic_get_and_set((int32*)lock, 1) == 0) break; } @@ -364,7 +364,7 @@ acquire_spinlock(spinlock* lock) } else { #if DEBUG_SPINLOCKS int32 oldValue; - oldValue = atomic_or((int32*)lock, 1); + oldValue = atomic_get_and_set((int32*)lock, 1); if (oldValue != 0) { panic("acquire_spinlock: attempt to acquire lock %p twice on " "non-SMP system (last caller: %p, value %" B_PRId32 ")", lock, @@ -407,13 +407,13 @@ acquire_spinlock_nocheck(spinlock *lock) PAUSE(); } - if (atomic_or((int32*)lock, 1) == 0) + if (atomic_get_and_set((int32*)lock, 1) == 0) break; } #endif } else { #if DEBUG_SPINLOCKS - if (atomic_or((int32*)lock, 1) != 0) { + if (atomic_get_and_set((int32*)lock, 1) != 0) { panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice " "on non-SMP system\n", lock); } @@ -450,7 +450,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) process_all_pending_ici(currentCPU); PAUSE(); } - if (atomic_or((int32*)lock, 1) == 0) + if (atomic_get_and_set((int32*)lock, 1) == 0) break; } @@ -461,7 +461,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) } else { #if DEBUG_SPINLOCKS int32 oldValue; - oldValue = atomic_or((int32*)lock, 1); + oldValue = atomic_get_and_set((int32*)lock, 1); if (oldValue != 0) { panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " "non-SMP system (last caller: %p, value %" B_PRId32 ")", lock, @@ -498,9 +498,11 @@ release_spinlock(spinlock *lock) } } } -#else - if (atomic_and((int32*)lock, 0) != 1) +#elif DEBUG_SPINLOCKS + if (atomic_get_and_set((int32*)lock, 0) != 1) panic("release_spinlock: lock %p was already released\n", lock); +#else + atomic_set((int32*)lock, 0); #endif } else { #if DEBUG_SPINLOCKS @@ -508,7 +510,7 @@ release_spinlock(spinlock *lock) panic("release_spinlock: attempt to release lock %p with " "interrupts enabled\n", lock); } - if (atomic_and((int32*)lock, 0) != 1) + if (atomic_get_and_set((int32*)lock, 0) != 1) panic("release_spinlock: lock %p was already released\n", lock); #endif #if DEBUG_SPINLOCK_LATENCIES From d929eb20ba2f9f884a9013b14b3fd85cbad4ba12 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 23 Jul 2013 22:24:15 +0200 Subject: [PATCH 099/273] docs: Update atomic_*() functions documentation --- docs/user/support/SupportDefs.dox | 90 ++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/docs/user/support/SupportDefs.dox b/docs/user/support/SupportDefs.dox index 8c29d42a89..962a9ccfc6 100644 --- a/docs/user/support/SupportDefs.dox +++ b/docs/user/support/SupportDefs.dox @@ -627,15 +627,13 @@ //! @{ -/*! \fn int32 atomic_set(vint32 *value, int32 newValue) +/*! \fn void atomic_set(int32* value, int32 newValue) \brief Atomically set the variable \a value to \a newvalue. This is a thread-safe way of performing the \c *value \c = \c newValue operation. You should use these function when two or more threads might access the variable simultaneously. You don't have to use a semaphore or a - mutex in this case. - - \return The original value of \c value. + mutex in this case. The variable must be naturally aligned. \sa atomic_set64() for a version that works on \c long \c long. \sa atomic_test_and_set() @@ -646,7 +644,28 @@ */ -/*! \fn int32 atomic_test_and_set(vint32 *value, int32 newValue, +/*! \fn int32 atomic_get_and_set(int32* value, int32 newValue) + \brief Atomically set the variable \a value to \a newvalue and return the + old value. + + This is a thread-safe way of performing the \c *value \c = \c newValue + operation. You should use these function when two or more threads might + access the variable simultaneously. You don't have to use a semaphore or a + mutex in this case. + + \return The original value of \c value. + + \sa atomic_get_and_set64() for a version that works on \c long \c long. + \sa atomic_set() + \sa atomic_test_and_set() + \sa atomic_add() + \sa atomic_and() + \sa atomic_or(), + \sa atomic_get() +*/ + + +/*! \fn int32 atomic_test_and_set(int32* value, int32 newValue, int32 testAgainst) \brief Atomically set the variable \a value to \a newValue if the current value is \a testAgainst. @@ -659,6 +678,7 @@ \return The original value of \c value. \sa atomic_test_and_set64() for a version that works on \c long \c long. + \sa atomic_get_and_set() \sa atomic_set() \sa atomic_add() \sa atomic_and() @@ -668,7 +688,7 @@ /*! - \fn int32 atomic_add(vint32 *value, int32 addValue) + \fn int32 atomic_add(int32* value, int32 addValue) \brief Atomically add the value of \a addValue to \a value. This is a thread-safe way of performing the \c *value \c += \c addValue @@ -679,6 +699,7 @@ \return The original value of \c value. \sa atomic_add64() for a version that works on \c long \c long. + \sa atomic_get_and_set() \sa atomic_set() \sa atomic_test_and_set() \sa atomic_and() @@ -687,7 +708,7 @@ */ -/*! \fn int32 atomic_and(vint32 *value, int32 andValue) +/*! \fn int32 atomic_and(int32* value, int32 andValue) \brief Atomically perform a bitwise AND operation of \a andValue to the variable \a andValue. @@ -699,6 +720,7 @@ \return The original value of \c value. \sa atomic_and64() for a version that works on \c long \c long. + \sa atomic_get_and_set() \sa atomic_set() \sa atomic_test_and_set() \sa atomic_add() @@ -709,7 +731,7 @@ /*! - \fn int32 atomic_or(vint32 *value, int32 orValue) + \fn int32 atomic_or(int32* value, int32 orValue) \brief Atomically perform a bitwise OR operation of \a orValue to the variable \a andValue. @@ -721,6 +743,7 @@ \return The original value of \c value. \sa atomic_or64() for a version that works on \c long \c long. + \sa atomic_get_and_set() \sa atomic_set() \sa atomic_test_and_set() \sa atomic_add() @@ -730,17 +753,18 @@ /*! - \fn int32 atomic_get(vint32 *value) + \fn int32 atomic_get(int32* value) \brief Atomically return the value of \c value. This is a thread-safe way of reading the contents of the \c value operation. You should use these function when two or more threads might access the variable simultaneously. You don't have to use a semaphore or a - mutex in this case. + mutex in this case. The variable must be naturally aligned. \return The original value of \c value. \sa atomic_get64() for a version that works on \c long \c long. + \sa atomic_get_and_set() \sa atomic_set() \sa atomic_test_and_set() \sa atomic_add() @@ -750,17 +774,16 @@ /*! - \fn int64 atomic_set64(vint64 *value, int64 newValue) + \fn void atomic_set64(int64* value, int64 newValue) \brief Atomically set the variable \a value to \a newvalue. This is a thread-safe way of performing the \c *value \c = \c newValue operation. You should use these function when two or more threads might access the variable simultaneously. You don't have to use a semaphore or a - mutex in this case. - - \return The original value of \c value. + mutex in this case. The variable must be naturally aligned. \sa atomic_set() for a version that works on an \c int32. + \sa atomic_get_and_set64() \sa atomic_test_and_set64() \sa atomic_add64() \sa atomic_and64() @@ -770,7 +793,29 @@ /*! - \fn int64 atomic_test_and_set64(vint64 *value, int64 newValue, + \fn int64 atomic_get_and_set64(int64* value, int64 newValue) + \brief Atomically set the variable \a value to \a newvalue and return + the old value. + + This is a thread-safe way of performing the \c *value \c = \c newValue + operation. You should use these function when two or more threads might + access the variable simultaneously. You don't have to use a semaphore or a + mutex in this case. + + \return The original value of \c value. + + \sa atomic_get_and_set() for a version that works on an \c int32. + \sa atomic_set64() + \sa atomic_test_and_set64() + \sa atomic_add64() + \sa atomic_and64() + \sa atomic_or64() + \sa atomic_get64() +*/ + + +/*! + \fn int64 atomic_test_and_set64(int64* value, int64 newValue, int64 testAgainst) \brief Atomically set the variable \a value to \a newValue if the current value is \a testAgainst. @@ -783,6 +828,7 @@ \return The original value of \c value. \sa atomic_test_and_set() for a version that works on an \c int32. + \sa atomic_get_and_set64() \sa atomic_set64() \sa atomic_add64() \sa atomic_and64() @@ -792,7 +838,7 @@ /*! - \fn int64 atomic_add64(vint64 *value, int64 addValue) + \fn int64 atomic_add64(int64* value, int64 addValue) \brief Atomically add the value of \a addValue to \a value. This is a thread-safe way of performing the \c *value \c += \c addValue @@ -803,6 +849,7 @@ \return The original value of \c value. \sa atomic_add() for a version that works on an \c int32. + \sa atomic_get_and_set64() \sa atomic_set64() \sa atomic_test_and_set64() \sa atomic_and64() @@ -812,7 +859,7 @@ /*! - \fn int64 atomic_and64(vint64 *value, int64 andValue) + \fn int64 atomic_and64(int64* value, int64 andValue) \brief Atomically perform a bitwise AND operation of \a andValue to the variable \a andValue. @@ -824,6 +871,7 @@ \return The original value of \c value. \sa atomic_and() for a version that works on an \c int32. + \sa atomic_get_and_set64() \sa atomic_set64() \sa atomic_test_and_set64() \sa atomic_add64() @@ -832,7 +880,7 @@ */ -/*! \fn int64 atomic_or64(vint64 *value, int64 orValue) +/*! \fn int64 atomic_or64(int64* value, int64 orValue) \brief Atomically perform a bitwise OR operation of \a orValue to the variable \a andValue. @@ -844,6 +892,7 @@ \return The original value of \c value. \sa atomic_or() for a version that works on an \c int32. + \sa atomic_get_and_set64() \sa atomic_set64() \sa atomic_test_and_set64() \sa atomic_add64() @@ -853,17 +902,18 @@ /*! - \fn int64 atomic_get64(vint64 *value) + \fn int64 atomic_get64(int64* value) \brief Atomically return the value of \c value. This is a thread-safe way of reading the contents of the \c value operation. You should use these function when two or more threads might access the variable simultaneously. You don't have to use a semaphore or a - mutex in this case. + mutex in this case. The variable must be naturally aligned. \return The original value of \c value. \sa atomic_get() for a version that works on an \c int32. + \sa atomic_get_and_set64() \sa atomic_set64() \sa atomic_test_and_set64() \sa atomic_add64() From 73ad2473e7874b3702cf5b0fdf4c81b747812ed9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 6 Nov 2013 00:03:07 +0100 Subject: [PATCH 100/273] Remove remaining unnecessary 'volatile' qualifiers --- headers/cpp/std/bastring.h | 4 ++-- headers/os/media/TimeSource.h | 2 +- headers/os/opengl/GLRenderer.h | 2 +- headers/private/graphics/intel_extreme/lock.h | 2 +- headers/private/graphics/radeon_hd/lock.h | 4 ++-- headers/private/kernel/MessagingServiceDefs.h | 2 +- headers/private/kernel/lock.h | 2 +- headers/private/kernel/thread_types.h | 2 +- headers/private/kernel/user_debugger.h | 2 +- headers/private/kernel/vm/vm_types.h | 2 +- headers/private/kernel/wait_for_objects.h | 4 ++-- headers/private/media/SharedBufferList.h | 2 +- headers/private/print/BeUtils.h | 2 +- .../bus_managers/acpi/EmbeddedController.h | 2 +- .../kernel/drivers/audio/hda/device.cpp | 2 +- src/add-ons/kernel/drivers/audio/hda/driver.h | 2 +- .../drivers/network/usb_asix/ASIXDevice.h | 2 +- .../network/usb_davicom/DavicomDevice.h | 2 +- .../drivers/network/usb_ecm/ECMDevice.h | 2 +- src/add-ons/kernel/file_systems/cdda/Lock.h | 2 +- .../kernel/file_systems/nfs4/RPCServer.h | 2 +- .../media/media-add-ons/usb_webcam/AddOn.cpp | 2 +- .../generic/support/Referenceable.h | 2 +- src/apps/showimage/Filter.h | 2 +- src/apps/showimage/ImageCache.h | 2 +- src/apps/terminal/TermParse.cpp | 6 +++--- src/apps/terminal/TermParse.h | 2 +- .../freebsd_wlan/net80211/ieee80211_haiku.cpp | 2 +- src/preferences/mail/DNSQuery.cpp | 2 +- src/servers/media/DefaultManager.h | 2 +- src/servers/print/PrintServerApp.h | 2 +- src/servers/print/Printer.h | 2 +- .../mime/MimeSnifferAddonManager.cpp | 2 +- .../registrar/mime/RegistrarThreadManager.h | 2 +- .../arch/x86/paging/X86PagingStructures.h | 4 ++-- src/system/kernel/debug/debug.cpp | 6 +++--- src/system/kernel/fs/Vnode.h | 2 +- src/system/kernel/locks/user_mutex.cpp | 4 ++-- src/system/kernel/posix/realtime_sem.cpp | 2 +- src/system/kernel/posix/xsi_semaphore.cpp | 4 ++-- src/system/kernel/timer.cpp | 8 ++++---- src/system/kernel/util/Random.cpp | 2 +- src/system/kernel/vm/vm_page.cpp | 19 ++++++++++--------- .../posix/glibc/include/bits/libc-lock.h | 2 +- .../libroot/posix/pthread/pthread_once.cpp | 6 +++--- 45 files changed, 69 insertions(+), 68 deletions(-) diff --git a/headers/cpp/std/bastring.h b/headers/cpp/std/bastring.h index 774f7be3e9..15bf4b5c86 100644 --- a/headers/cpp/std/bastring.h +++ b/headers/cpp/std/bastring.h @@ -66,7 +66,7 @@ extern void __length_error (const char *); #endif #ifdef __HAIKU__ -extern "C" __haiku_int32 atomic_add(volatile __haiku_int32* value, +extern "C" __haiku_int32 atomic_add(__haiku_int32* value, __haiku_int32 addvalue); #endif /* __HAIKU__ */ @@ -82,7 +82,7 @@ private: charT* data () { return reinterpret_cast(this + 1); } charT& operator[] (size_t s) { return data () [s]; } #ifdef __HAIKU__ - charT* grab () { if (selfish) return clone (); atomic_add((volatile __haiku_int32*) &ref, 1); return data (); } + charT* grab () { if (selfish) return clone (); atomic_add((__haiku_int32*) &ref, 1); return data (); } void release() { if (atomic_add((__haiku_int32*) &ref, -1) == 1) delete this; } #else charT* grab () { if (selfish) return clone (); ++ref; return data (); } diff --git a/headers/os/media/TimeSource.h b/headers/os/media/TimeSource.h index cd8a783dfb..7a76e54b26 100644 --- a/headers/os/media/TimeSource.h +++ b/headers/os/media/TimeSource.h @@ -113,7 +113,7 @@ private: private: bool fStarted; area_id fArea; - volatile BPrivate::media::TimeSourceTransmit* fBuf; + BPrivate::media::TimeSourceTransmit* fBuf; BPrivate::media::SlaveNodes* fSlaveNodes; area_id _reserved_area; diff --git a/headers/os/opengl/GLRenderer.h b/headers/os/opengl/GLRenderer.h index 9bdbd865df..2d9c6ac4de 100644 --- a/headers/os/opengl/GLRenderer.h +++ b/headers/os/opengl/GLRenderer.h @@ -54,7 +54,7 @@ private: virtual status_t _Reserved_Renderer_3(int32, void *); virtual status_t _Reserved_Renderer_4(int32, void *); - volatile int32 fRefCount; // How much we're still usefull? + int32 fRefCount; // How much we're still usefull? BGLView* fView; // Never forget who is the boss! ulong fOptions; // Keep that tune in memory BGLDispatcher* fDispatcher; // Our personal OpenGL API call dispatcher diff --git a/headers/private/graphics/intel_extreme/lock.h b/headers/private/graphics/intel_extreme/lock.h index 9c0f8423fa..255e177f94 100644 --- a/headers/private/graphics/intel_extreme/lock.h +++ b/headers/private/graphics/intel_extreme/lock.h @@ -14,7 +14,7 @@ typedef struct lock { sem_id sem; - vint32 count; + int32 count; } lock; diff --git a/headers/private/graphics/radeon_hd/lock.h b/headers/private/graphics/radeon_hd/lock.h index e76c4e4f45..9557bc98b5 100644 --- a/headers/private/graphics/radeon_hd/lock.h +++ b/headers/private/graphics/radeon_hd/lock.h @@ -14,7 +14,7 @@ typedef struct lock { sem_id sem; - vint32 count; + int32 count; } lock; @@ -82,4 +82,4 @@ class Autolock { }; -#endif /* LOCK_H */ \ No newline at end of file +#endif /* LOCK_H */ diff --git a/headers/private/kernel/MessagingServiceDefs.h b/headers/private/kernel/MessagingServiceDefs.h index 81a353c0ed..9d1c40aede 100644 --- a/headers/private/kernel/MessagingServiceDefs.h +++ b/headers/private/kernel/MessagingServiceDefs.h @@ -17,7 +17,7 @@ enum { }; struct messaging_area_header { - vint32 lock_counter; + int32 lock_counter; int32 size; // set to 0, when area is discarded area_id kernel_area; area_id next_kernel_area; diff --git a/headers/private/kernel/lock.h b/headers/private/kernel/lock.h index bf4c039566..64667a72f7 100644 --- a/headers/private/kernel/lock.h +++ b/headers/private/kernel/lock.h @@ -47,7 +47,7 @@ typedef struct rw_lock { struct rw_lock_waiter* waiters; spinlock lock; thread_id holder; - vint32 count; + int32 count; int32 owner_count; int16 active_readers; // Only > 0 while a writer is waiting: number diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 9afb2f516d..792d693082 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -406,7 +406,7 @@ private: TeamTimeUserTimerList fCPUTimeUserTimers; // protected by scheduler lock TeamUserTimeUserTimerList fUserTimeUserTimers; - vint32 fUserDefinedTimerCount; // accessed atomically + int32 fUserDefinedTimerCount; // accessed atomically }; diff --git a/headers/private/kernel/user_debugger.h b/headers/private/kernel/user_debugger.h index 777483388b..809289c445 100644 --- a/headers/private/kernel/user_debugger.h +++ b/headers/private/kernel/user_debugger.h @@ -68,7 +68,7 @@ struct team_debug_info { thread_id causing_thread; // thread that caused the debugger to be attached; -1 for manual // debugger attachment (or no debugger installed) - vint32 image_event; + int32 image_event; // counter incremented whenever an image is created/deleted struct ConditionVariable* debugger_changed_condition; diff --git a/headers/private/kernel/vm/vm_types.h b/headers/private/kernel/vm/vm_types.h index 43c4f80cf8..85aa030687 100644 --- a/headers/private/kernel/vm/vm_types.h +++ b/headers/private/kernel/vm/vm_types.h @@ -136,7 +136,7 @@ public: #endif #if DEBUG_PAGE_ACCESS - vint32 accessing_thread; + int32 accessing_thread; #endif #if VM_PAGE_ALLOCATION_TRACKING_AVAILABLE diff --git a/headers/private/kernel/wait_for_objects.h b/headers/private/kernel/wait_for_objects.h index 8d1b0add5b..30cd6f7f0e 100644 --- a/headers/private/kernel/wait_for_objects.h +++ b/headers/private/kernel/wait_for_objects.h @@ -17,12 +17,12 @@ struct select_sync; typedef struct select_info { struct select_info* next; // next in the object's list struct select_sync* sync; - vint32 events; + int32 events; uint16 selected_events; } select_info; typedef struct select_sync { - vint32 ref_count; + int32 ref_count; sem_id sem; uint32 count; struct select_info* set; diff --git a/headers/private/media/SharedBufferList.h b/headers/private/media/SharedBufferList.h index 37f8a7efb3..c5b93b131b 100644 --- a/headers/private/media/SharedBufferList.h +++ b/headers/private/media/SharedBufferList.h @@ -54,7 +54,7 @@ private: private: sem_id fSemaphore; - vint32 fAtom; + int32 fAtom; _shared_buffer_info fInfos[kMaxBuffers]; int32 fCount; diff --git a/headers/private/print/BeUtils.h b/headers/private/print/BeUtils.h index 3e1f25be79..a294c3bde4 100644 --- a/headers/private/print/BeUtils.h +++ b/headers/private/print/BeUtils.h @@ -53,7 +53,7 @@ status_t TestForAddonExistence(const char* name, directory_which which, // Reference counted object class Object { private: - volatile int32 fRefCount; + int32 fRefCount; public: // After construction reference count is 1 diff --git a/src/add-ons/kernel/bus_managers/acpi/EmbeddedController.h b/src/add-ons/kernel/bus_managers/acpi/EmbeddedController.h index c9ad95d58e..2ac30c64cd 100644 --- a/src/add-ons/kernel/bus_managers/acpi/EmbeddedController.h +++ b/src/add-ons/kernel/bus_managers/acpi/EmbeddedController.h @@ -151,7 +151,7 @@ struct acpi_ec_cookie { mutex ec_lock; int ec_burstactive; int ec_sci_pending; - vint32 ec_gencount; + int32 ec_gencount; ConditionVariable ec_condition_var; int ec_suspending; }; diff --git a/src/add-ons/kernel/drivers/audio/hda/device.cpp b/src/add-ons/kernel/drivers/audio/hda/device.cpp index aeb09966eb..f114f0ffc7 100644 --- a/src/add-ons/kernel/drivers/audio/hda/device.cpp +++ b/src/add-ons/kernel/drivers/audio/hda/device.cpp @@ -25,7 +25,7 @@ hda_open(const char* name, uint32 flags, void** cookie) if (controller == NULL) return ENODEV; - if (controller->opened != 0) + if (atomic_get(&controller->opened) != 0) return B_BUSY; status_t status = hda_hw_init(controller); diff --git a/src/add-ons/kernel/drivers/audio/hda/driver.h b/src/add-ons/kernel/drivers/audio/hda/driver.h index fc848b2cb1..18c06b0a04 100644 --- a/src/add-ons/kernel/drivers/audio/hda/driver.h +++ b/src/add-ons/kernel/drivers/audio/hda/driver.h @@ -65,7 +65,7 @@ struct hda_multi; */ struct hda_controller { struct pci_info pci_info; - vint32 opened; + int32 opened; const char* devfs_path; area_id regs_area; diff --git a/src/add-ons/kernel/drivers/network/usb_asix/ASIXDevice.h b/src/add-ons/kernel/drivers/network/usb_asix/ASIXDevice.h index 0bccbbfb68..5aa13faf93 100644 --- a/src/add-ons/kernel/drivers/network/usb_asix/ASIXDevice.h +++ b/src/add-ons/kernel/drivers/network/usb_asix/ASIXDevice.h @@ -97,7 +97,7 @@ virtual status_t ReadMACAddress(ether_address_t *address); bool fRemoved; bool fHasConnection; bool fNonBlocking; - vint32 fInsideNotify; + int32 fInsideNotify; // interface and device infos uint16 fFrameSize; diff --git a/src/add-ons/kernel/drivers/network/usb_davicom/DavicomDevice.h b/src/add-ons/kernel/drivers/network/usb_davicom/DavicomDevice.h index af0ce0e0de..233eef2f85 100644 --- a/src/add-ons/kernel/drivers/network/usb_davicom/DavicomDevice.h +++ b/src/add-ons/kernel/drivers/network/usb_davicom/DavicomDevice.h @@ -162,7 +162,7 @@ static void _NotifyCallback(void *cookie, int32 status, bool fHasConnection; bool fTXBufferFull; bool fNonBlocking; - vint32 fInsideNotify; + int32 fInsideNotify; // pipes for notifications, data io and tx packet size usb_pipe fNotifyEndpoint; diff --git a/src/add-ons/kernel/drivers/network/usb_ecm/ECMDevice.h b/src/add-ons/kernel/drivers/network/usb_ecm/ECMDevice.h index 2cc5b266d0..bfbd4e4b35 100644 --- a/src/add-ons/kernel/drivers/network/usb_ecm/ECMDevice.h +++ b/src/add-ons/kernel/drivers/network/usb_ecm/ECMDevice.h @@ -45,7 +45,7 @@ static void _NotifyCallback(void *cookie, int32 status, status_t fStatus; bool fOpen; bool fRemoved; - vint32 fInsideNotify; + int32 fInsideNotify; usb_device fDevice; uint16 fVendorID; uint16 fProductID; diff --git a/src/add-ons/kernel/file_systems/cdda/Lock.h b/src/add-ons/kernel/file_systems/cdda/Lock.h index e6db2727c5..4c66e6b51d 100644 --- a/src/add-ons/kernel/file_systems/cdda/Lock.h +++ b/src/add-ons/kernel/file_systems/cdda/Lock.h @@ -63,7 +63,7 @@ class Semaphore { private: sem_id fSemaphore; #ifdef USE_BENAPHORE - vint32 fCount; + int32 fCount; #endif }; diff --git a/src/add-ons/kernel/file_systems/nfs4/RPCServer.h b/src/add-ons/kernel/file_systems/nfs4/RPCServer.h index 35c52f8bc1..50e218b813 100644 --- a/src/add-ons/kernel/file_systems/nfs4/RPCServer.h +++ b/src/add-ons/kernel/file_systems/nfs4/RPCServer.h @@ -107,7 +107,7 @@ private: uint32 fRepairCount; mutex fRepairLock; - vint32 fXID; + int32 fXID; }; diff --git a/src/add-ons/media/media-add-ons/usb_webcam/AddOn.cpp b/src/add-ons/media/media-add-ons/usb_webcam/AddOn.cpp index a115c3f854..13fdb8b9df 100644 --- a/src/add-ons/media/media-add-ons/usb_webcam/AddOn.cpp +++ b/src/add-ons/media/media-add-ons/usb_webcam/AddOn.cpp @@ -179,7 +179,7 @@ WebCamMediaAddOn::FillDefaultFlavorInfo(flavor_info* info) info->info = "USB Web Camera"; info->kinds = B_BUFFER_PRODUCER | B_CONTROLLABLE | B_PHYSICAL_INPUT; info->flavor_flags = 0;//B_FLAVOR_IS_GLOBAL; - info->internal_id = atomic_add((vint32 *)&fInternalIDCounter, 1); + info->internal_id = atomic_add((int32*)&fInternalIDCounter, 1); info->possible_count = 1;//0; info->in_format_count = 0; info->in_format_flags = 0; diff --git a/src/apps/icon-o-matic/generic/support/Referenceable.h b/src/apps/icon-o-matic/generic/support/Referenceable.h index 00cdf55438..6539324e9f 100644 --- a/src/apps/icon-o-matic/generic/support/Referenceable.h +++ b/src/apps/icon-o-matic/generic/support/Referenceable.h @@ -21,7 +21,7 @@ class Referenceable { bool Release(); private: - vint32 fReferenceCount; + int32 fReferenceCount; }; #endif // REFERENCABLE_H diff --git a/src/apps/showimage/Filter.h b/src/apps/showimage/Filter.h index 8021ca133f..7b410ebf0e 100644 --- a/src/apps/showimage/Filter.h +++ b/src/apps/showimage/Filter.h @@ -149,7 +149,7 @@ private: bool fStarted; // has Start() been called? sem_id fWaitForThreads; // to exit int32 fN; // the number of used filter threads - volatile int32 fNumberOfThreads; // the current number of FilterThreads + int32 fNumberOfThreads; // the current number of FilterThreads volatile bool fIsRunning; // FilterThreads should process data as long as it is true BBitmap* fSrcImage; bool fDestImageInitialized; diff --git a/src/apps/showimage/ImageCache.h b/src/apps/showimage/ImageCache.h index 000fa95ff2..ac572c02fa 100644 --- a/src/apps/showimage/ImageCache.h +++ b/src/apps/showimage/ImageCache.h @@ -86,7 +86,7 @@ private: CacheList fCacheEntriesByAge; QueueMap fQueueMap; QueueDeque fQueue; - vint32 fThreadCount; + int32 fThreadCount; int32 fMaxThreadCount; uint64 fBytes; uint64 fMaxBytes; diff --git a/src/apps/terminal/TermParse.cpp b/src/apps/terminal/TermParse.cpp index b86fbe52f3..d22ef38154 100644 --- a/src/apps/terminal/TermParse.cpp +++ b/src/apps/terminal/TermParse.cpp @@ -1205,9 +1205,9 @@ TermParse::_ReadParserBuffer() fBuffer->Unlock(); // wait for new input from pty - if (fReadBufferSize == 0) { + if (atomic_get(&fReadBufferSize) == 0) { status_t status = B_OK; - while (fReadBufferSize == 0 && status == B_OK) { + while (atomic_get(&fReadBufferSize) == 0 && status == B_OK) { do { status = acquire_sem(fReaderSem); } while (status == B_INTERRUPTED); @@ -1224,7 +1224,7 @@ TermParse::_ReadParserBuffer() } } - int32 toRead = fReadBufferSize; + int32 toRead = atomic_get(&fReadBufferSize); if (toRead > ESC_PARSER_BUFFER_SIZE) toRead = ESC_PARSER_BUFFER_SIZE; diff --git a/src/apps/terminal/TermParse.h b/src/apps/terminal/TermParse.h index a9aeb8dacd..26e57ecc78 100644 --- a/src/apps/terminal/TermParse.h +++ b/src/apps/terminal/TermParse.h @@ -73,7 +73,7 @@ private: uint fBufferPosition; uchar fReadBuffer[READ_BUF_SIZE]; - vint32 fReadBufferSize; + int32 fReadBufferSize; uchar fParserBuffer[ESC_PARSER_BUFFER_SIZE]; int32 fParserBufferSize; diff --git a/src/libs/compat/freebsd_wlan/net80211/ieee80211_haiku.cpp b/src/libs/compat/freebsd_wlan/net80211/ieee80211_haiku.cpp index b1f936febb..e0398ffa5b 100644 --- a/src/libs/compat/freebsd_wlan/net80211/ieee80211_haiku.cpp +++ b/src/libs/compat/freebsd_wlan/net80211/ieee80211_haiku.cpp @@ -463,7 +463,7 @@ int ieee80211_node_dectestref(struct ieee80211_node* ni) { // atomic_add returns old value - return atomic_add((vint32*)&ni->ni_refcnt, -1) == 1; + return atomic_add((int32*)&ni->ni_refcnt, -1) == 1; } diff --git a/src/preferences/mail/DNSQuery.cpp b/src/preferences/mail/DNSQuery.cpp index 4b638ea548..d2a018de01 100644 --- a/src/preferences/mail/DNSQuery.cpp +++ b/src/preferences/mail/DNSQuery.cpp @@ -19,7 +19,7 @@ #endif -static vint32 gID = 1; +static int32 gID = 1; BRawNetBuffer::BRawNetBuffer() diff --git a/src/servers/media/DefaultManager.h b/src/servers/media/DefaultManager.h index 33a78c01e6..826b31d063 100644 --- a/src/servers/media/DefaultManager.h +++ b/src/servers/media/DefaultManager.h @@ -74,7 +74,7 @@ private: uint32 fBeginHeader[3]; uint32 fEndHeader[3]; thread_id fRescanThread; - vint32 fRescanRequested; + int32 fRescanRequested; BLocker fRescanLock; }; diff --git a/src/servers/print/PrintServerApp.h b/src/servers/print/PrintServerApp.h index 55e8447d99..3c242aac0b 100644 --- a/src/servers/print/PrintServerApp.h +++ b/src/servers/print/PrintServerApp.h @@ -89,7 +89,7 @@ class PrintServerApp : public BApplication, public FolderListener { BBitmap *fSelectedIconMini; BBitmap *fSelectedIconLarge; #endif - vint32 fReferences; + int32 fReferences; sem_id fHasReferences; Settings *fSettings; bool fUseConfigWindow; diff --git a/src/servers/print/Printer.h b/src/servers/print/Printer.h index 07dbff878f..c15b676837 100644 --- a/src/servers/print/Printer.h +++ b/src/servers/print/Printer.h @@ -110,7 +110,7 @@ private: // the next job to process Job* fJob; // the current nmber of processing threads - vint32 fProcessing; + int32 fProcessing; // stop processing bool fAbort; static BObjectList sPrinters; diff --git a/src/servers/registrar/mime/MimeSnifferAddonManager.cpp b/src/servers/registrar/mime/MimeSnifferAddonManager.cpp index 5f3d433ba0..8c5e7bf887 100644 --- a/src/servers/registrar/mime/MimeSnifferAddonManager.cpp +++ b/src/servers/registrar/mime/MimeSnifferAddonManager.cpp @@ -50,7 +50,7 @@ struct MimeSnifferAddonManager::AddonReference { private: BMimeSnifferAddon* fAddon; - vint32 fReferenceCount; + int32 fReferenceCount; }; diff --git a/src/servers/registrar/mime/RegistrarThreadManager.h b/src/servers/registrar/mime/RegistrarThreadManager.h index e111bcde54..6dcc59ccd5 100644 --- a/src/servers/registrar/mime/RegistrarThreadManager.h +++ b/src/servers/registrar/mime/RegistrarThreadManager.h @@ -39,7 +39,7 @@ private: std::list::iterator& RemoveThread(std::list::iterator &i); std::list fThreads; - vint32 fThreadCount; + int32 fThreadCount; }; #endif // THREAD_MANAGER_H diff --git a/src/system/kernel/arch/x86/paging/X86PagingStructures.h b/src/system/kernel/arch/x86/paging/X86PagingStructures.h index fe8b9d9a94..6fd882b39f 100644 --- a/src/system/kernel/arch/x86/paging/X86PagingStructures.h +++ b/src/system/kernel/arch/x86/paging/X86PagingStructures.h @@ -17,8 +17,8 @@ struct X86PagingStructures : DeferredDeletable { phys_addr_t pgdir_phys; - vint32 ref_count; - vint32 active_on_cpus; + int32 ref_count; + int32 active_on_cpus; // mask indicating on which CPUs the map is currently used X86PagingStructures(); diff --git a/src/system/kernel/debug/debug.cpp b/src/system/kernel/debug/debug.cpp index 15adaba629..e75acd66dc 100644 --- a/src/system/kernel/debug/debug.cpp +++ b/src/system/kernel/debug/debug.cpp @@ -139,10 +139,10 @@ static int32 sCurrentLine = 0; static debugger_demangle_module_info* sDemangleModule; static Thread* sDebuggedThread; -static vint32 sInDebugger = 0; +static int32 sInDebugger = 0; static bool sPreviousDprintfState; static volatile bool sHandOverKDL = false; -static vint32 sHandOverKDLToCPU = -1; +static int32 sHandOverKDLToCPU = -1; static bool sCPUTrapped[B_MAX_CPU_COUNT]; @@ -1007,7 +1007,7 @@ hand_over_kernel_debugger() // hand over to another CPU without us noticing. Since this is only // initiated by the user, it is harmless, though. sHandOverKDL = true; - while (sHandOverKDLToCPU >= 0) + while (atomic_get(&sHandOverKDLToCPU) >= 0) PAUSE(); } diff --git a/src/system/kernel/fs/Vnode.h b/src/system/kernel/fs/Vnode.h index a40cf65059..d4ef8fd3ff 100644 --- a/src/system/kernel/fs/Vnode.h +++ b/src/system/kernel/fs/Vnode.h @@ -104,7 +104,7 @@ private: void _WakeUpLocker(); private: - vint32 fFlags; + int32 fFlags; static Bucket sBuckets[kBucketCount]; }; diff --git a/src/system/kernel/locks/user_mutex.cpp b/src/system/kernel/locks/user_mutex.cpp index d842bfcc41..42e2d933c0 100644 --- a/src/system/kernel/locks/user_mutex.cpp +++ b/src/system/kernel/locks/user_mutex.cpp @@ -99,7 +99,7 @@ remove_user_mutex_entry(UserMutexEntry* entry) static status_t -user_mutex_lock_locked(vint32* mutex, addr_t physicalAddress, const char* name, +user_mutex_lock_locked(int32* mutex, addr_t physicalAddress, const char* name, uint32 flags, bigtime_t timeout, MutexLocker& locker) { // mark the mutex locked + waiting @@ -148,7 +148,7 @@ user_mutex_lock_locked(vint32* mutex, addr_t physicalAddress, const char* name, static void -user_mutex_unlock_locked(vint32* mutex, addr_t physicalAddress, uint32 flags) +user_mutex_unlock_locked(int32* mutex, addr_t physicalAddress, uint32 flags) { if (UserMutexEntry* entry = sUserMutexTable.Lookup(physicalAddress)) { // Someone is waiting -- set the locked flag. It might still be set, diff --git a/src/system/kernel/posix/realtime_sem.cpp b/src/system/kernel/posix/realtime_sem.cpp index 2e25c53569..7c51e80666 100644 --- a/src/system/kernel/posix/realtime_sem.cpp +++ b/src/system/kernel/posix/realtime_sem.cpp @@ -141,7 +141,7 @@ public: private: char* fName; - vint32 fRefCount; + int32 fRefCount; uid_t fUID; gid_t fGID; mode_t fPermissions; diff --git a/src/system/kernel/posix/xsi_semaphore.cpp b/src/system/kernel/posix/xsi_semaphore.cpp index 4c76c799cf..a80548a663 100644 --- a/src/system/kernel/posix/xsi_semaphore.cpp +++ b/src/system/kernel/posix/xsi_semaphore.cpp @@ -631,8 +631,8 @@ static mutex sIpcLock; static mutex sXsiSemaphoreSetLock; static uint32 sGlobalSequenceNumber = 1; -static vint32 sXsiSemaphoreCount = 0; -static vint32 sXsiSemaphoreSetCount = 0; +static int32 sXsiSemaphoreCount = 0; +static int32 sXsiSemaphoreSetCount = 0; // #pragma mark - diff --git a/src/system/kernel/timer.cpp b/src/system/kernel/timer.cpp index afea3a79d2..d10e3c3fa0 100644 --- a/src/system/kernel/timer.cpp +++ b/src/system/kernel/timer.cpp @@ -29,7 +29,7 @@ struct per_cpu_timer_data { spinlock lock; timer* volatile events; timer* volatile current_event; - vint32 current_event_in_progress; + int32 current_event_in_progress; bigtime_t real_time_offset; }; @@ -264,7 +264,7 @@ timer_interrupt() cpuData.events = (timer*)event->next; cpuData.current_event = event; - cpuData.current_event_in_progress = 1; + atomic_set(&cpuData.current_event_in_progress, 1); release_spinlock(spinlock); @@ -277,7 +277,7 @@ timer_interrupt() if (event->hook) rc = event->hook(event); - cpuData.current_event_in_progress = 0; + atomic_set(&cpuData.current_event_in_progress, 0); acquire_spinlock(spinlock); @@ -447,7 +447,7 @@ cancel_timer(timer* event) if (cpu != smp_get_current_cpu()) { spinLocker.Unlock(); - while (cpuData.current_event_in_progress == 1) { + while (atomic_get(&cpuData.current_event_in_progress) == 1) { PAUSE(); } } diff --git a/src/system/kernel/util/Random.cpp b/src/system/kernel/util/Random.cpp index 12a2eeb074..c4de06a0f9 100644 --- a/src/system/kernel/util/Random.cpp +++ b/src/system/kernel/util/Random.cpp @@ -109,7 +109,7 @@ random_value() unsigned int secure_random_value() { - static vint32 count = 0; + static int32 count = 0; uint32 data[8]; data[0] = atomic_add(&count, 1); diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index dd9b9b3d48..9f1f548887 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -119,9 +119,9 @@ static page_num_t sNonExistingPages; static uint64 sIgnoredPages; // pages of physical memory ignored by the boot loader (and thus not // available here) -static vint32 sUnreservedFreePages; -static vint32 sUnsatisfiedPageReservations; -static vint32 sModifiedTemporaryPages; +static int32 sUnreservedFreePages; +static int32 sUnsatisfiedPageReservations; +static int32 sModifiedTemporaryPages; static ConditionVariable sFreePageCondition; static mutex sPageDeficitLock = MUTEX_INITIALIZER("page deficit"); @@ -1407,7 +1407,7 @@ static uint32 reserve_some_pages(uint32 count, uint32 dontTouch) { while (true) { - int32 freePages = sUnreservedFreePages; + int32 freePages = atomic_get(&sUnreservedFreePages); if (freePages <= (int32)dontTouch) return 0; @@ -1456,7 +1456,7 @@ static inline void unreserve_pages(uint32 count) { atomic_add(&sUnreservedFreePages, count); - if (sUnsatisfiedPageReservations != 0) + if (atomic_get(&sUnsatisfiedPageReservations) != 0) wake_up_page_reservation_waiters(); } @@ -1744,7 +1744,8 @@ page_scrubber(void *unused) snooze(100000); // 100ms if (sFreePageQueue.Count() == 0 - || sUnreservedFreePages < (int32)sFreePagesTarget) { + || atomic_get(&sUnreservedFreePages) + < (int32)sFreePagesTarget) { continue; } @@ -1880,7 +1881,7 @@ private: uint32 fMaxPages; uint32 fWrapperCount; uint32 fTransferCount; - vint32 fPendingTransfers; + int32 fPendingTransfers; PageWriteWrapper* fWrappers; PageWriteTransfer* fTransfers; ConditionVariable fAllFinishedCondition; @@ -2206,7 +2207,7 @@ PageWriterRun::AddPage(vm_page* page) uint32 PageWriterRun::Go() { - fPendingTransfers = fTransferCount; + atomic_set(&fPendingTransfers, fTransferCount); fAllFinishedCondition.Init(this, "page writer wait for I/O"); ConditionVariableEntry waitEntry; @@ -3005,7 +3006,7 @@ reserve_pages(uint32 count, int priority, bool dontWait) bool notifyDaemon = sUnsatisfiedPageReservations == 0; sUnsatisfiedPageReservations += count; - if (sUnreservedFreePages > dontTouch) { + if (atomic_get(&sUnreservedFreePages) > dontTouch) { // the situation changed sUnsatisfiedPageReservations -= count; continue; diff --git a/src/system/libroot/posix/glibc/include/bits/libc-lock.h b/src/system/libroot/posix/glibc/include/bits/libc-lock.h index 5a1b5d1811..fc460360e4 100644 --- a/src/system/libroot/posix/glibc/include/bits/libc-lock.h +++ b/src/system/libroot/posix/glibc/include/bits/libc-lock.h @@ -19,7 +19,7 @@ extern char _single_threaded; static inline int __compare_and_swap (volatile int32 *p, int oldval, int newval) { - int32 readval = atomic_test_and_set(p, newval, oldval); + int32 readval = atomic_test_and_set((int32*)p, newval, oldval); return (readval == oldval ? 1 : 0); } diff --git a/src/system/libroot/posix/pthread/pthread_once.cpp b/src/system/libroot/posix/pthread/pthread_once.cpp index 72926b8fc9..7ef275452b 100644 --- a/src/system/libroot/posix/pthread/pthread_once.cpp +++ b/src/system/libroot/posix/pthread/pthread_once.cpp @@ -55,7 +55,7 @@ pthread_once(pthread_once_t* onceControl, void (*initRoutine)(void)) // initRoutine. All following threads will return right away. while (true) { - int32 value = atomic_test_and_set((vint32*)&onceControl->state, + int32 value = atomic_test_and_set((int32*)&onceControl->state, STATE_INITIALIZING, STATE_UNINITIALIZED); if (value == STATE_INITIALIZED) @@ -83,7 +83,7 @@ pthread_once(pthread_once_t* onceControl, void (*initRoutine)(void)) sem_id semaphore = create_sem(0, "pthread once"); if (semaphore >= 0) { // successfully created -- set it - value = atomic_test_and_set((vint32*)&onceControl->state, + value = atomic_test_and_set((int32*)&onceControl->state, semaphore, STATE_INITIALIZING); if (value == STATE_INITIALIZING) value = semaphore; @@ -93,7 +93,7 @@ pthread_once(pthread_once_t* onceControl, void (*initRoutine)(void)) // Failed to create the semaphore. Can only happen when the // system runs out of semaphores, but we can still handle the // situation gracefully by spinning. - value = atomic_test_and_set((vint32*)&onceControl->state, + value = atomic_test_and_set((int32*)&onceControl->state, STATE_SPINNING, STATE_INITIALIZING); if (value == STATE_INITIALIZING) value = STATE_SPINNING; From aa4aca0264aa285e07c0b8e43112ac06803fcd2e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 01:32:48 +0100 Subject: [PATCH 101/273] kernel: Protect signal data with Team::signal_lock --- headers/private/kernel/thread_types.h | 11 ++- src/system/kernel/UserEvent.cpp | 7 +- src/system/kernel/signal.cpp | 117 ++++++++++++++++---------- src/system/kernel/team.cpp | 2 + src/system/kernel/thread.cpp | 11 ++- 5 files changed, 93 insertions(+), 55 deletions(-) diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 792d693082..4ae7ad823a 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -291,6 +291,8 @@ struct Team : TeamThreadIteratorEntry, KernelReferenceable, bool initialized; // true when the state has been initialized } exit; + spinlock signal_lock; + public: ~Team(); @@ -398,7 +400,7 @@ private: BKernel::QueuedSignalsCounter* fQueuedSignalsCounter; BKernel::PendingSignals fPendingSignals; - // protected by scheduler lock + // protected by signal_lock struct sigaction fSignalActions[MAX_SIGNAL_NUMBER]; // indexed signal - 1, protected by fLock @@ -429,7 +431,7 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, int32 pinned_to_cpu; // only accessed by this thread or in the // scheduler, when thread is not running - sigset_t sig_block_mask; // protected by scheduler lock, + sigset_t sig_block_mask; // protected by team->signal_lock, // only modified by the thread itself sigset_t sigsuspend_original_unblocked_mask; // non-0 after a return from _user_sigsuspend(), containing the inverted @@ -481,7 +483,8 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, /* this field may only stay in debug builds in the future */ BKernel::Team *team; // protected by team lock, thread lock, scheduler - // lock + // lock, team_lock + spinlock team_lock; struct { sem_id sem; // immutable after thread creation @@ -604,7 +607,7 @@ private: mutex fLock; BKernel::PendingSignals fPendingSignals; - // protected by scheduler lock + // protected by team->signal_lock UserTimerList fUserTimers; // protected by fLock ThreadTimeUserTimerList fCPUTimeUserTimers; diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp index 9e460a572c..751cebf1e3 100644 --- a/src/system/kernel/UserEvent.cpp +++ b/src/system/kernel/UserEvent.cpp @@ -115,7 +115,7 @@ TeamSignalEvent::Fire() fSignal->AcquireReference(); // one reference is transferred to send_signal_to_team_locked - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(fTeam->signal_lock); status_t error = send_signal_to_team_locked(fTeam, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); locker.Unlock(); @@ -169,11 +169,12 @@ ThreadSignalEvent::Fire() fSignal->AcquireReference(); // one reference is transferred to send_signal_to_team_locked - - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker teamLocker(fThread->team_lock); + SpinLocker locker(fThread->team->signal_lock); status_t error = send_signal_to_thread_locked(fThread, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); locker.Unlock(); + teamLocker.Unlock(); // There are situations (for certain signals), in which // send_signal_to_team_locked() succeeds without queuing the signal. diff --git a/src/system/kernel/signal.cpp b/src/system/kernel/signal.cpp index 751d763f6e..fe41dfea9a 100644 --- a/src/system/kernel/signal.cpp +++ b/src/system/kernel/signal.cpp @@ -746,7 +746,7 @@ class SigSuspendDone : public AbstractTraceEntry { /*! Updates the given thread's Thread::flags field according to what signals are pending. - The caller must hold the scheduler lock. + The caller must hold \c team->signal_lock. */ static void update_thread_signals_flag(Thread* thread) @@ -761,7 +761,7 @@ update_thread_signals_flag(Thread* thread) /*! Updates the current thread's Thread::flags field according to what signals are pending. - The caller must hold the scheduler lock. + The caller must hold \c team->signal_lock. */ static void update_current_thread_signals_flag() @@ -772,7 +772,7 @@ update_current_thread_signals_flag() /*! Updates all of the given team's threads' Thread::flags fields according to what signals are pending. - The caller must hold the scheduler lock. + The caller must hold \c signal_lock. */ static void update_team_threads_signal_flag(Team* team) @@ -824,7 +824,7 @@ notify_debugger(Thread* thread, Signal* signal, struct sigaction& handler, After dequeuing the signal the Thread::flags field of the affected threads are updated. The caller gets a reference to the returned signal, if any. - The caller must hold the scheduler lock. + The caller must hold \c team->signal_lock. \param thread The thread. \param nonBlocked The mask of non-blocked signals. \param buffer If the signal is not queued this buffer is returned. In this @@ -916,7 +916,7 @@ handle_signals(Thread* thread) Team* team = thread->team; TeamLocker teamLocker(team); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->team->signal_lock); // If userland requested to defer signals, we check now, if this is // possible. @@ -947,7 +947,7 @@ handle_signals(Thread* thread) initialIteration = false; } else { teamLocker.Lock(); - schedulerLocker.Lock(); + locker.Lock(); signalMask = thread->AllPendingSignals() & nonBlockedMask; } @@ -957,7 +957,7 @@ handle_signals(Thread* thread) if ((signalMask & KILL_SIGNALS) == 0 && (atomic_get(&thread->debug_info.flags) & B_THREAD_DEBUG_STOP) != 0) { - schedulerLocker.Unlock(); + locker.Unlock(); teamLocker.Unlock(); user_debug_stop_thread(); @@ -976,7 +976,7 @@ handle_signals(Thread* thread) ASSERT(signal != NULL); SignalHandledCaller signalHandledCaller(signal); - schedulerLocker.Unlock(); + locker.Unlock(); // get the action for the signal struct sigaction handler; @@ -1122,13 +1122,16 @@ handle_signals(Thread* thread) // Suspend the thread, unless there's already a signal to // continue or kill pending. - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - if ((thread->AllPendingSignals() - & (CONTINUE_SIGNALS | KILL_SIGNALS)) == 0) { + locker.Lock(); + bool resume = (thread->AllPendingSignals() + & (CONTINUE_SIGNALS | KILL_SIGNALS)) != 0; + locker.Unlock(); + + if (!resume) { + InterruptsSpinLocker schedulerLocker(gSchedulerLock); thread->next_state = B_THREAD_SUSPENDED; scheduler_reschedule(); } - schedulerLocker.Unlock(); continue; } @@ -1215,7 +1218,7 @@ handle_signals(Thread* thread) TRACE(("### Setting up custom signal handler frame...\n")); // save the old block mask -- we may need to adjust it for the handler - schedulerLocker.Lock(); + locker.Lock(); sigset_t oldBlockMask = thread->sigsuspend_original_unblocked_mask != 0 ? ~thread->sigsuspend_original_unblocked_mask @@ -1232,7 +1235,7 @@ handle_signals(Thread* thread) update_current_thread_signals_flag(); - schedulerLocker.Unlock(); + locker.Unlock(); setup_signal_frame(thread, &handler, signal, oldBlockMask); @@ -1263,7 +1266,7 @@ handle_signals(Thread* thread) /*! Checks whether the given signal is blocked for the given team (i.e. all of its threads). - The caller must hold the team's lock and the scheduler lock. + The caller must hold the team's lock and \c signal_lock. */ bool is_team_signal_blocked(Team* team, int signal) @@ -1308,7 +1311,7 @@ signal_get_user_stack(addr_t address, stack_t* stack) /*! Checks whether any non-blocked signal is pending for the current thread. - The caller must hold the scheduler lock. + The caller must hold \c team->signal_lock. \param thread The current thread. */ static bool @@ -1338,7 +1341,7 @@ has_permission_to_signal(Signal* signal, Team* team) /*! Delivers a signal to the \a thread, but doesn't handle the signal -- it just makes sure the thread gets the signal, i.e. unblocks it if needed. - The caller must hold the scheduler lock. + The caller must hold \c team->signal_lock. \param thread The thread the signal shall be delivered to. \param signalNumber The number of the signal to be delivered. If \c 0, no @@ -1374,6 +1377,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, if (thread->team == team_get_kernel_team()) { // Signals to kernel threads will only wake them up + SpinLocker _(gSchedulerLock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); return B_OK; @@ -1397,10 +1401,12 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, mainThread->AddPendingSignal(SIGKILLTHR); // wake up main thread + SpinLocker locker(gSchedulerLock); if (mainThread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(mainThread); else thread_interrupt(mainThread, true); + locker.Unlock(); update_thread_signals_flag(mainThread); } @@ -1408,24 +1414,31 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, // supposed to fall through } case SIGKILLTHR: + { // Wake up suspended threads and interrupt waiting ones + SpinLocker locker(gSchedulerLock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); else thread_interrupt(thread, true); - break; + break; + } case SIGNAL_CONTINUE_THREAD: + { // wake up thread, and interrupt its current syscall + SpinLocker locker(gSchedulerLock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); atomic_or(&thread->flags, THREAD_FLAGS_DONT_RESTART_SYSCALL); break; - + } case SIGCONT: + { // Wake up thread if it was suspended, otherwise interrupt it, if // the signal isn't blocked. + SpinLocker locker(gSchedulerLock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); else if ((SIGNAL_TO_MASK(SIGCONT) & ~thread->sig_block_mask) != 0) @@ -1434,7 +1447,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, // remove any pending stop signals thread->RemovePendingSignals(STOP_SIGNALS); break; - + } default: // If the signal is not masked, interrupt the thread, if it is // currently waiting (interruptibly). @@ -1442,6 +1455,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, & (~thread->sig_block_mask | SIGNAL_TO_MASK(SIGCHLD))) != 0) { // Interrupt thread if it was waiting + SpinLocker locker(gSchedulerLock); thread_interrupt(thread, false); } break; @@ -1455,8 +1469,6 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, /*! Sends the given signal to the given thread. - The caller must not hold the scheduler lock. - \param thread The thread the signal shall be sent to. \param signal The signal to be delivered. If the signal's number is \c 0, no actual signal will be delivered. Only delivery checks will be performed. @@ -1482,15 +1494,19 @@ send_signal_to_thread(Thread* thread, const Signal& signal, uint32 flags) if (error != B_OK) return error; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker teamLocker(thread->team_lock); + SpinLocker locker(thread->team->signal_lock); error = send_signal_to_thread_locked(thread, signal.Number(), signalToQueue, flags); if (error != B_OK) return error; + locker.Unlock(); + teamLocker.Unlock(); + if ((flags & B_DO_NOT_RESCHEDULE) == 0) - scheduler_reschedule_if_necessary_locked(); + scheduler_reschedule_if_necessary(); return B_OK; } @@ -1498,8 +1514,6 @@ send_signal_to_thread(Thread* thread, const Signal& signal, uint32 flags) /*! Sends the given signal to the thread with the given ID. - The caller must not hold the scheduler lock. - \param threadID The ID of the thread the signal shall be sent to. \param signal The signal to be delivered. If the signal's number is \c 0, no actual signal will be delivered. Only delivery checks will be performed. @@ -1528,7 +1542,7 @@ send_signal_to_thread_id(thread_id threadID, const Signal& signal, uint32 flags) /*! Sends the given signal to the given team. - The caller must hold the scheduler lock. + The caller must hold \c signal_lock. \param team The team the signal shall be sent to. \param signalNumber The number of the signal to be delivered. If \c 0, no @@ -1591,6 +1605,7 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, mainThread->AddPendingSignal(SIGKILLTHR); // wake up main thread + SpinLocker _(gSchedulerLock); if (mainThread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(mainThread); else @@ -1604,6 +1619,7 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, // don't block the signal. for (Thread* thread = team->thread_list; thread != NULL; thread = thread->team_next) { + SpinLocker _(gSchedulerLock); if (thread->state == B_THREAD_SUSPENDED) { scheduler_enqueue_in_run_queue(thread); } else if ((SIGNAL_TO_MASK(SIGCONT) & ~thread->sig_block_mask) @@ -1645,17 +1661,16 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, thread = thread->team_next) { sigset_t nonBlocked = ~thread->sig_block_mask | SIGNAL_TO_MASK(SIGCHLD); - if ((thread->AllPendingSignals() & nonBlocked) != 0) + if ((thread->AllPendingSignals() & nonBlocked) != 0) { + SpinLocker _(gSchedulerLock); thread_interrupt(thread, false); + } } break; } update_team_threads_signal_flag(team); - if ((flags & B_DO_NOT_RESCHEDULE) == 0) - scheduler_reschedule_if_necessary_locked(); - return B_OK; } @@ -1687,10 +1702,17 @@ send_signal_to_team(Team* team, const Signal& signal, uint32 flags) if (error != B_OK) return error; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(team->signal_lock); - return send_signal_to_team_locked(team, signal.Number(), signalToQueue, - flags); + error = send_signal_to_team_locked(team, signal.Number(), signalToQueue, + flags); + + locker.Unlock(); + + if ((flags & B_DO_NOT_RESCHEDULE) == 0) + scheduler_reschedule_if_necessary(); + + return error; } @@ -1876,7 +1898,7 @@ sigprocmask_internal(int how, const sigset_t* set, sigset_t* oldSet) { Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker _(thread->team->signal_lock); sigset_t oldMask = thread->sig_block_mask; @@ -1947,7 +1969,7 @@ sigaction_internal(int signal, const struct sigaction* act, if ((act && act->sa_handler == SIG_IGN) || (act && act->sa_handler == SIG_DFL && (SIGNAL_TO_MASK(signal) & DEFAULT_IGNORE_SIGNALS) != 0)) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(team->signal_lock); team->RemovePendingSignal(signal); @@ -1989,7 +2011,7 @@ sigwait_internal(const sigset_t* set, siginfo_t* info, uint32 flags, Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->team->signal_lock); bool timedOut = false; status_t error = B_OK; @@ -2009,7 +2031,7 @@ sigwait_internal(const sigset_t* set, siginfo_t* info, uint32 flags, ASSERT(signal != NULL); SignalHandledCaller signalHandledCaller(signal); - schedulerLocker.Unlock(); + locker.Unlock(); info->si_signo = signal->Number(); info->si_code = signal->SignalCode(); @@ -2041,7 +2063,7 @@ sigwait_internal(const sigset_t* set, siginfo_t* info, uint32 flags, thread_prepare_to_block(thread, flags, THREAD_BLOCK_TYPE_SIGNAL, NULL); - schedulerLocker.Unlock(); + locker.Unlock(); if ((flags & B_ABSOLUTE_TIMEOUT) != 0) { error = thread_block_with_timeout(flags, timeout); @@ -2050,13 +2072,13 @@ sigwait_internal(const sigset_t* set, siginfo_t* info, uint32 flags, // POSIX requires EAGAIN (B_WOULD_BLOCK) on timeout timedOut = true; - schedulerLocker.Lock(); + locker.Lock(); break; } } else thread_block(); - schedulerLocker.Lock(); + locker.Lock(); } // restore the original block mask @@ -2082,7 +2104,7 @@ sigsuspend_internal(const sigset_t* _mask) Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->team->signal_lock); // Set the new block mask and block until interrupted. We might be here // after a syscall restart, in which case sigsuspend_original_unblocked_mask @@ -2096,7 +2118,10 @@ sigsuspend_internal(const sigset_t* _mask) while (!has_signals_pending(thread)) { thread_prepare_to_block(thread, B_CAN_INTERRUPT, THREAD_BLOCK_TYPE_SIGNAL, NULL); - thread_block_locked(thread); + + locker.Unlock(); + thread_block(); + locker.Lock(); } // Set sigsuspend_original_unblocked_mask (guaranteed to be non-0 due to @@ -2121,7 +2146,7 @@ sigpending_internal(sigset_t* set) if (set == NULL) return B_BAD_VALUE; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->team->signal_lock); *set = thread->AllPendingSignals() & thread->sig_block_mask; @@ -2412,13 +2437,13 @@ _user_restore_signal_frame(struct signal_frame_data* userSignalFrameData) } // restore the signal block mask - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->team->signal_lock); thread->sig_block_mask = signalFrameData.context.uc_sigmask & BLOCKABLE_SIGNALS; update_current_thread_signals_flag(); - schedulerLocker.Unlock(); + locker.Unlock(); // restore the syscall restart related thread flags and the syscall restart // parameters diff --git a/src/system/kernel/team.cpp b/src/system/kernel/team.cpp index c5b0f7e898..906ca0b64e 100644 --- a/src/system/kernel/team.cpp +++ b/src/system/kernel/team.cpp @@ -490,6 +490,8 @@ Team::Team(team_id id, bool kernel) // init dead/stopped/continued children condition vars dead_children.condition_variable.Init(&dead_children, "team children"); + B_INITIALIZE_SPINLOCK(&signal_lock); + fQueuedSignalsCounter = new(std::nothrow) BKernel::QueuedSignalsCounter( kernel ? -1 : MAX_QUEUED_SIGNALS); memset(fSignalActions, 0, sizeof(fSignalActions)); diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index d36b212a09..2125fc1e2e 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -206,6 +206,7 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) mutex_init_etc(&fLock, lockName, MUTEX_FLAG_CLONE_NAME); B_INITIALIZE_SPINLOCK(&time_lock); + B_INITIALIZE_SPINLOCK(&team_lock); // init name if (name != NULL) @@ -1103,11 +1104,13 @@ undertaker(void* /*args*/) Team* kernelTeam = team_get_kernel_team(); TeamLocker kernelTeamLocker(kernelTeam); thread->Lock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker signalLocker(kernelTeam->signal_lock); + SpinLocker schedulerLocker(gSchedulerLock); remove_thread_from_team(kernelTeam, thread); schedulerLocker.Unlock(); + signalLocker.Unlock(); kernelTeamLocker.Unlock(); // free the thread structure @@ -1986,6 +1989,7 @@ thread_exit(void) // swap address spaces, to make sure we're running on the kernel's pgdir vm_swap_address_space(team->address_space, VMAddressSpace::Kernel()); + SpinLocker teamLocker(thread->team_lock); SpinLocker schedulerLocker(gSchedulerLock); // removing the thread and putting its death entry to the parent // team needs to be an atomic operation @@ -2014,6 +2018,8 @@ thread_exit(void) remove_thread_from_team(team, thread); insert_thread_into_team(kernelTeam, thread); + teamLocker.Unlock(); + if (team->death_entry != NULL) { if (--team->death_entry->remaining_threads == 0) team->death_entry->condition.NotifyOne(true, B_OK); @@ -3282,7 +3288,8 @@ _user_cancel_thread(thread_id threadID, void (*cancelFunction)(int)) thread->cancel_function = cancelFunction; // send the cancellation signal to the thread - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker teamLocker(thread->team_lock); + SpinLocker locker(thread->team->signal_lock); return send_signal_to_thread_locked(thread, SIGNAL_CANCEL_THREAD, NULL, 0); } From 83983eaf38ac442edaa4418de4b6a389e911c72d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 01:40:02 +0100 Subject: [PATCH 102/273] kernel: Remove Thread::alarm --- headers/private/kernel/thread_types.h | 1 - src/system/kernel/thread.cpp | 7 ------- 2 files changed, 8 deletions(-) diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 4ae7ad823a..ec1d53f3c3 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -420,7 +420,6 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, int64 serial_number; // immutable after adding thread to hash Thread *hash_next; // protected by thread hash lock Thread *team_next; // protected by team lock and fLock - timer alarm; // protected by scheduler lock char name[B_OS_NAME_LENGTH]; // protected by fLock int32 priority; // protected by scheduler lock int32 io_priority; // protected by fLock diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 2125fc1e2e..e3b4a58954 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -214,8 +214,6 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) else strcpy(this->name, "unnamed thread"); - alarm.period = 0; - exit.status = 0; list_init(&exit.waiters); @@ -1925,9 +1923,6 @@ thread_exit(void) } if (team != kernelTeam) { - // Cancel previously installed alarm timer, if any. - cancel_timer(&thread->alarm); - // Delete all user timers associated with the thread. ThreadLocker threadLocker(thread); thread->DeleteUserTimers(false); @@ -2345,8 +2340,6 @@ thread_reset_for_exec(void) // reset thread CPU time clock thread->cpu_clock_offset = -thread->CPUTime(false); - - // Note: We don't cancel an alarm. It is supposed to survive exec*(). } From d3e5752b112000b3722d744259be44fa8438cfe7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 01:50:20 +0100 Subject: [PATCH 103/273] scheduler: Performance mode is actually low latency mode --- headers/private/kernel/kscheduler.h | 2 +- src/system/kernel/scheduler/scheduler.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index bfdac1d7e2..c1eeb39c1d 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -18,7 +18,7 @@ struct SchedulerListener; typedef enum scheduler_mode { - SCHEDULER_MODE_PERFORMANCE, + SCHEDULER_MODE_LOW_LATENCY, SCHEDULER_MODE_POWER_SAVING, // ... SCHEDULER_MODE_COUNT diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 87f564a5a3..e113e25dde 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -481,7 +481,7 @@ has_cache_expired(Thread* thread) CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; switch (sSchedulerMode) { - case SCHEDULER_MODE_PERFORMANCE: + case SCHEDULER_MODE_LOW_LATENCY: return coreEntry->fActiveTime - schedulerThreadData->went_sleep_active > kCacheExpire; @@ -708,7 +708,7 @@ update_priority_heaps(int32 cpu, int32 priority) static int32 -choose_core_performance(Thread* thread) +choose_core_low_latency(Thread* thread) { CoreEntry* entry; @@ -800,7 +800,7 @@ choose_cpu(int32 core) static bool -should_rebalance_performance(Thread* thread) +should_rebalance_low_latency(Thread* thread) { scheduler_thread_data* schedulerThreadData = thread->scheduler_data; ASSERT(schedulerThreadData->previous_core >= 0); @@ -1575,7 +1575,7 @@ scheduler_start(void) status_t scheduler_set_operation_mode(scheduler_mode mode) { - if (mode != SCHEDULER_MODE_PERFORMANCE + if (mode != SCHEDULER_MODE_LOW_LATENCY && mode != SCHEDULER_MODE_POWER_SAVING) { return B_BAD_VALUE; } @@ -1587,11 +1587,11 @@ scheduler_set_operation_mode(scheduler_mode mode) sSchedulerMode = mode; switch (mode) { - case SCHEDULER_MODE_PERFORMANCE: + case SCHEDULER_MODE_LOW_LATENCY: sDisableSmallTaskPacking = -1; sSmallTaskCore = -1; - sChooseCore = choose_core_performance; - sShouldRebalance = should_rebalance_performance; + sChooseCore = choose_core_low_latency; + sShouldRebalance = should_rebalance_low_latency; break; case SCHEDULER_MODE_POWER_SAVING: @@ -1802,7 +1802,7 @@ _scheduler_init() } #if 1 - scheduler_set_operation_mode(SCHEDULER_MODE_PERFORMANCE); + scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); #else scheduler_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); #endif From 20ded5c2eb7b51da4e34c40078de2903243ef56f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 02:06:42 +0100 Subject: [PATCH 104/273] kernel/posix: Do not use thread_block_locked() --- src/system/kernel/posix/xsi_message_queue.cpp | 3 +-- src/system/kernel/posix/xsi_semaphore.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/system/kernel/posix/xsi_message_queue.cpp b/src/system/kernel/posix/xsi_message_queue.cpp index 247a1aeda1..00046cb906 100644 --- a/src/system/kernel/posix/xsi_message_queue.cpp +++ b/src/system/kernel/posix/xsi_message_queue.cpp @@ -126,12 +126,11 @@ public: // Unlock the queue before blocking queueLocker->Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); // TODO: We've got a serious race condition: If BlockAndUnlock() returned due to // interruption, we will still be queued. A WakeUpThread() at this point will // call thread_unblock() and might thus screw with our trying to re-lock the // mutex. - return thread_block_locked(thread); + return thread_block(); } void DoIpcSet(struct msqid_ds *result) diff --git a/src/system/kernel/posix/xsi_semaphore.cpp b/src/system/kernel/posix/xsi_semaphore.cpp index a80548a663..5a48f9d28d 100644 --- a/src/system/kernel/posix/xsi_semaphore.cpp +++ b/src/system/kernel/posix/xsi_semaphore.cpp @@ -142,12 +142,11 @@ public: // Unlock the set before blocking setLocker->Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); // TODO: We've got a serious race condition: If BlockAndUnlock() returned due to // interruption, we will still be queued. A WakeUpThread() at this point will // call thread_unblock() and might thus screw with our trying to re-lock the // mutex. - return thread_block_locked(thread); + return thread_block(); } void Deque(queued_thread *queueEntry, bool waitForZero) From defee266db232f7477d62a5ff8f10a0a498cad1e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 04:20:32 +0100 Subject: [PATCH 105/273] kernel: Add read write spinlock implementation --- headers/os/drivers/KernelExport.h | 16 +++++ headers/private/kernel/smp.h | 59 ++++++++++++++++ headers/private/kernel/util/AutoLock.h | 96 ++++++++++++++++++++++++++ src/system/kernel/smp.cpp | 87 +++++++++++++++++++++++ 4 files changed, 258 insertions(+) diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index d4168557dd..de40d0c76d 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -40,6 +40,15 @@ typedef ulong cpu_status; # define B_SPINLOCK_IS_LOCKED(lock) (*(lock) > 0) #endif +typedef struct { + int32 lock; +} rw_spinlock; + +#define B_RW_SPINLOCK_INITIALIZER { 0 } +#define B_INITIALIZE_RW_SPINLOCK(rw_spinlock) do { \ + (rw_spinlock)->lock = 0; \ + } while (false) + typedef struct { spinlock lock; uint32 count; @@ -137,6 +146,13 @@ extern void restore_interrupts(cpu_status status); extern void acquire_spinlock(spinlock *lock); extern void release_spinlock(spinlock *lock); +extern bool try_acquire_write_spinlock(rw_spinlock* lock); +extern void acquire_write_spinlock(rw_spinlock* lock); +extern void release_write_spinlock(rw_spinlock* lock); +extern bool try_acquire_read_spinlock(rw_spinlock* lock); +extern void acquire_read_spinlock(rw_spinlock* lock); +extern void release_read_spinlock(rw_spinlock* lock); + extern bool try_acquire_write_seqlock(seqlock* lock); extern void acquire_write_seqlock(seqlock* lock); extern void release_write_seqlock(seqlock* lock); diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index c7aa163762..8ef0e82a34 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -103,6 +103,65 @@ release_spinlock_inline(spinlock* lock) #endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION +static inline bool +try_acquire_write_spinlock_inline(rw_spinlock* lock) +{ + return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; +} + + +static inline void +acquire_write_spinlock_inline(rw_spinlock* lock) +{ + if (try_acquire_write_spinlock(lock)) + return; + acquire_write_spinlock(lock); +} + + +static inline void +release_write_spinlock_inline(rw_spinlock* lock) +{ + atomic_set(&lock->lock, 0); +} + + +static inline bool +try_acquire_read_spinlock_inline(rw_spinlock* lock) +{ + uint32 previous = atomic_add(&lock->lock, 1); + if ((previous & (1 << 31)) == 0) + return true; + atomic_test_and_set(&lock->lock, 1 << 31, previous); + return false; +} + + +static inline void +acquire_read_spinlock_inline(rw_spinlock* lock) +{ + if (try_acquire_read_spinlock(lock)) + return; + acquire_read_spinlock(lock); +} + + +static inline void +release_read_spinlock_inline(rw_spinlock* lock) +{ + atomic_add(&lock->lock, -1); +} + + +#define try_acquire_read_spinlock(lock) try_acquire_read_spinlock_inline(lock) +#define acquire_read_spinlock(lock) acquire_read_spinlock_inline(lock) +#define release_read_spinlock(lock) release_read_spinlock_inline(lock) +#define try_acquire_write_spinlock(lock) \ + try_acquire_write_spinlock(lock) +#define acquire_write_spinlock(lock) acquire_write_spinlock_inline(lock) +#define release_write_spinlock(lock) release_write_spinlock_inline(lock) + + static inline bool try_acquire_write_seqlock_inline(seqlock* lock) { bool succeed = try_acquire_spinlock(&lock->lock); diff --git a/headers/private/kernel/util/AutoLock.h b/headers/private/kernel/util/AutoLock.h index 5a334fbc1a..3151065857 100644 --- a/headers/private/kernel/util/AutoLock.h +++ b/headers/private/kernel/util/AutoLock.h @@ -160,6 +160,98 @@ private: typedef AutoLocker InterruptsSpinLocker; +class ReadSpinLocking { +public: + inline bool Lock(rw_spinlock* lockable) + { + acquire_read_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_read_spinlock(lockable); + } +}; + +typedef AutoLocker ReadSpinLocker; + + +class InterruptsReadSpinLocking { +public: + InterruptsReadSpinLocking() + : + fState(0) + { + } + + inline bool Lock(rw_spinlock* lockable) + { + fState = disable_interrupts(); + acquire_read_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_read_spinlock(lockable); + restore_interrupts(fState); + } + +private: + int fState; +}; + +typedef AutoLocker + InterruptsReadSpinLocker; + + +class WriteSpinLocking { +public: + inline bool Lock(rw_spinlock* lockable) + { + acquire_write_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_write_spinlock(lockable); + } +}; + +typedef AutoLocker WriteSpinLocker; + + +class InterruptsWriteSpinLocking { +public: + InterruptsWriteSpinLocking() + : + fState(0) + { + } + + inline bool Lock(rw_spinlock* lockable) + { + fState = disable_interrupts(); + acquire_write_spinlock(lockable); + return true; + } + + inline void Unlock(rw_spinlock* lockable) + { + release_write_spinlock(lockable); + restore_interrupts(fState); + } + +private: + int fState; +}; + +typedef AutoLocker + InterruptsWriteSpinLocker; + + class WriteSequentialLocking { public: inline bool Lock(seqlock* lockable) @@ -237,6 +329,10 @@ using BPrivate::WriteLocker; using BPrivate::InterruptsLocker; using BPrivate::SpinLocker; using BPrivate::InterruptsSpinLocker; +using BPrivate::ReadSpinLocker; +using BPrivate::InterruptsReadSpinLocker; +using BPrivate::WriteSpinLocker; +using BPrivate::InterruptsWriteSpinLocker; using BPrivate::WriteSequentialLocker; using BPrivate::InterruptsWriteSequentialLocker; using BPrivate::ThreadCPUPinner; diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index caf29bfe9a..950b499a9b 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -45,6 +45,13 @@ #undef acquire_spinlock #undef release_spinlock +#undef try_acquire_read_spinlock +#undef acquire_read_spinlock +#undef release_read_spinlock +#undef try_acquire_write_spinlock +#undef acquire_write_spinlock +#undef release_write_spinlock + #undef try_acquire_write_seqlock #undef acquire_write_seqlock #undef release_write_seqlock @@ -520,6 +527,86 @@ release_spinlock(spinlock *lock) } +bool +try_acquire_write_spinlock(rw_spinlock* lock) +{ + return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; +} + + +void +acquire_write_spinlock(rw_spinlock* lock) +{ + if (sNumCPUs < 2) + return; + + uint32 count = 0; + int currentCPU = smp_get_current_cpu(); + while (true) { + if (try_acquire_write_spinlock(lock)) + break; + + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_write_spinlock(): Failed to acquire spinlock %p " + "for a long time!", lock); + count = 0; + } + + process_all_pending_ici(currentCPU); + PAUSE(); + } +} + + +void +release_write_spinlock(rw_spinlock* lock) +{ + atomic_set(&lock->lock, 0); +} + + +bool +try_acquire_read_spinlock(rw_spinlock* lock) +{ + uint32 previous = atomic_add(&lock->lock, 1); + if ((previous & (1 << 31)) == 0) + return true; + atomic_test_and_set(&lock->lock, 1 << 31, previous); + return false; +} + + +void +acquire_read_spinlock(rw_spinlock* lock) +{ + if (sNumCPUs < 2) + return; + + uint32 count = 0; + int currentCPU = smp_get_current_cpu(); + while (1) { + if (try_acquire_read_spinlock(lock)) + break; + + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_read_spinlock(): Failed to acquire spinlock %p " + "for a long time!", lock); + count = 0; + } + + process_all_pending_ici(currentCPU); + PAUSE(); + } +} + + +void +release_read_spinlock(rw_spinlock* lock) +{ + atomic_add(&lock->lock, -1); +} + + bool try_acquire_write_seqlock(seqlock* lock) { bool succeed = try_acquire_spinlock(&lock->lock); From 3519eb334ac161e88fa56bcef287a2275a1c963e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 04:20:59 +0100 Subject: [PATCH 106/273] kernel: Change Thread::team_lock to rw_spinlock --- headers/private/kernel/thread_types.h | 2 +- src/system/kernel/UserEvent.cpp | 2 +- src/system/kernel/signal.cpp | 2 +- src/system/kernel/thread.cpp | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index ec1d53f3c3..96014a22fd 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -483,7 +483,7 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, BKernel::Team *team; // protected by team lock, thread lock, scheduler // lock, team_lock - spinlock team_lock; + rw_spinlock team_lock; struct { sem_id sem; // immutable after thread creation diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp index 751cebf1e3..f0c49b6f58 100644 --- a/src/system/kernel/UserEvent.cpp +++ b/src/system/kernel/UserEvent.cpp @@ -169,7 +169,7 @@ ThreadSignalEvent::Fire() fSignal->AcquireReference(); // one reference is transferred to send_signal_to_team_locked - InterruptsSpinLocker teamLocker(fThread->team_lock); + InterruptsReadSpinLocker teamLocker(fThread->team_lock); SpinLocker locker(fThread->team->signal_lock); status_t error = send_signal_to_thread_locked(fThread, fSignal->Number(), fSignal, B_DO_NOT_RESCHEDULE); diff --git a/src/system/kernel/signal.cpp b/src/system/kernel/signal.cpp index fe41dfea9a..7907dfa9b4 100644 --- a/src/system/kernel/signal.cpp +++ b/src/system/kernel/signal.cpp @@ -1494,7 +1494,7 @@ send_signal_to_thread(Thread* thread, const Signal& signal, uint32 flags) if (error != B_OK) return error; - InterruptsSpinLocker teamLocker(thread->team_lock); + InterruptsReadSpinLocker teamLocker(thread->team_lock); SpinLocker locker(thread->team->signal_lock); error = send_signal_to_thread_locked(thread, signal.Number(), signalToQueue, diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index e3b4a58954..fa63e16d01 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -206,7 +206,7 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) mutex_init_etc(&fLock, lockName, MUTEX_FLAG_CLONE_NAME); B_INITIALIZE_SPINLOCK(&time_lock); - B_INITIALIZE_SPINLOCK(&team_lock); + B_INITIALIZE_RW_SPINLOCK(&team_lock); // init name if (name != NULL) @@ -1984,7 +1984,7 @@ thread_exit(void) // swap address spaces, to make sure we're running on the kernel's pgdir vm_swap_address_space(team->address_space, VMAddressSpace::Kernel()); - SpinLocker teamLocker(thread->team_lock); + WriteSpinLocker teamLocker(thread->team_lock); SpinLocker schedulerLocker(gSchedulerLock); // removing the thread and putting its death entry to the parent // team needs to be an atomic operation @@ -3281,7 +3281,7 @@ _user_cancel_thread(thread_id threadID, void (*cancelFunction)(int)) thread->cancel_function = cancelFunction; // send the cancellation signal to the thread - InterruptsSpinLocker teamLocker(thread->team_lock); + InterruptsReadSpinLocker teamLocker(thread->team_lock); SpinLocker locker(thread->team->signal_lock); return send_signal_to_thread_locked(thread, SIGNAL_CANCEL_THREAD, NULL, 0); } From 72addc62e042d2a39fba1f04e2cc8dadc155adfe Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 7 Nov 2013 22:16:36 +0100 Subject: [PATCH 107/273] kernel: Introduce Thread::time_lock and Team::time_lock --- headers/private/kernel/thread_types.h | 11 +-- src/system/kernel/UserTimer.cpp | 74 ++++++++++--------- .../kernel/scheduler/scheduler_common.h | 12 +++ src/system/kernel/team.cpp | 13 ++-- src/system/kernel/thread.cpp | 20 +++-- 5 files changed, 78 insertions(+), 52 deletions(-) diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 96014a22fd..c0448213e3 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -264,10 +264,11 @@ struct Team : TeamThreadIteratorEntry, KernelReferenceable, struct team_debug_info debug_info; - // protected by scheduler lock + // protected by time_lock bigtime_t dead_threads_kernel_time; bigtime_t dead_threads_user_time; bigtime_t cpu_clock_offset; + spinlock time_lock; // user group information; protected by fLock, the *_uid/*_gid fields also // by the scheduler lock @@ -516,7 +517,7 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, bigtime_t user_time; // protected by time_lock bigtime_t kernel_time; // protected by time_lock bigtime_t last_time; // protected by time_lock - bigtime_t cpu_clock_offset; // protected by scheduler lock + bigtime_t cpu_clock_offset; // protected by time_lock void (*post_interrupt_callback)(void*); void* post_interrupt_data; @@ -610,7 +611,7 @@ private: UserTimerList fUserTimers; // protected by fLock ThreadTimeUserTimerList fCPUTimeUserTimers; - // protected by scheduler lock + // protected by time_lock }; @@ -749,7 +750,7 @@ Thread::DequeuePendingSignal(sigset_t nonBlocked, Signal& buffer) /*! Returns the thread's current total CPU time (kernel + user + offset). - The caller must hold the scheduler lock. + The caller must hold \c time_lock. \param ignoreCurrentRun If \c true and the thread is currently running, don't add the time since the last time \c last_time was updated. Should @@ -764,7 +765,7 @@ Thread::CPUTime(bool ignoreCurrentRun) const // If currently running, also add the time since the last check, unless // requested otherwise. - if (!ignoreCurrentRun && cpu != NULL) + if (!ignoreCurrentRun && last_time != 0) time += system_time() - last_time; return time; diff --git a/src/system/kernel/UserTimer.cpp b/src/system/kernel/UserTimer.cpp index 6c04125774..b3da871849 100644 --- a/src/system/kernel/UserTimer.cpp +++ b/src/system/kernel/UserTimer.cpp @@ -137,8 +137,6 @@ UserTimer::~UserTimer() Cancels the timer, if it is already scheduled, and optionally schedules it with new parameters. - The caller must not hold the scheduler lock. - \param nextTime The time at which the timer should go off the next time. If \c B_INFINITE_TIMEOUT, the timer will not be scheduled. Whether the value is interpreted as absolute or relative time, depends on \c flags. @@ -160,8 +158,6 @@ UserTimer::~UserTimer() /*! Cancels the timer, if it is scheduled. - - The caller must not hold the scheduler lock. */ void UserTimer::Cancel() @@ -176,8 +172,6 @@ UserTimer::Cancel() uint32& _overrunCount) Return information on the current timer. - The caller must not hold the scheduler lock. - \param _remainingTime Return variable that will be set to the microseconds remaining to the time for which the timer was scheduled next before the call. If it wasn't scheduled, the variable is set to @@ -513,7 +507,7 @@ TeamTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { InterruptsWriteSequentialLocker locker(sUserTimerLock); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + SpinLocker timeLocker(fTeam != NULL ? &fTeam->time_lock : NULL); // get the current time, but only if needed bool nowValid = fTeam != NULL; @@ -547,9 +541,13 @@ TeamTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, // Get the team. If it doesn't exist anymore, just don't schedule the // timer anymore. - fTeam = Team::Get(fTeamID); - if (fTeam == NULL) + Team* newTeam = Team::Get(fTeamID); + if (newTeam == NULL) { + fTeam = NULL; return; + } else if (fTeam == NULL) + timeLocker.SetTo(newTeam->time_lock, false); + fTeam = newTeam; fAbsolute = (flags & B_RELATIVE_TIMEOUT) == 0; @@ -576,7 +574,7 @@ TeamTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, count = acquire_read_seqlock(&sUserTimerLock); if (fTeam != NULL) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(fTeam->time_lock); _remainingTime = fNextTime - fTeam->CPUTime(false); _interval = fInterval; } else { @@ -591,7 +589,7 @@ TeamTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, /*! Deactivates the timer, if it is activated. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. */ void TeamTimeUserTimer::Deactivate() @@ -619,7 +617,7 @@ TeamTimeUserTimer::Deactivate() was just set. Schedules a kernel timer for the remaining time, respectively cancels it. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. \param unscheduledThread If not \c NULL, this is the thread that is currently running and which is in the process of being unscheduled. @@ -646,7 +644,7 @@ TeamTimeUserTimer::Update(Thread* unscheduledThread) /*! Called when the team's CPU time clock which this timer refers to has been set. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. \param changedBy The value by which the clock has changed. */ @@ -689,7 +687,7 @@ TeamTimeUserTimer::HandleTimer() /*! Schedules/cancels the kernel timer as necessary. \c fRunningThreads must be up-to-date. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. \param unscheduling \c true, when the current thread is in the process of being unscheduled. @@ -757,7 +755,7 @@ TeamUserTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { InterruptsWriteSequentialLocker locker(sUserTimerLock); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + SpinLocker timeLocker(fTeam != NULL ? &fTeam->time_lock : NULL); // get the current time, but only if needed bool nowValid = fTeam != NULL; @@ -786,9 +784,13 @@ TeamUserTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, // Get the team. If it doesn't exist anymore, just don't schedule the // timer anymore. - fTeam = Team::Get(fTeamID); - if (fTeam == NULL) + Team* newTeam = Team::Get(fTeamID); + if (newTeam == NULL) { + fTeam = NULL; return; + } else if (fTeam == NULL) + timeLocker.SetTo(newTeam->time_lock, false); + fTeam = newTeam; // convert relative to absolute timeouts if ((flags & B_RELATIVE_TIMEOUT) != 0) { @@ -813,7 +815,7 @@ TeamUserTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, count = acquire_read_seqlock(&sUserTimerLock); if (fTeam != NULL) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(fTeam->time_lock); _remainingTime = fNextTime - fTeam->UserCPUTime(); _interval = fInterval; } else { @@ -828,7 +830,7 @@ TeamUserTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, /*! Deactivates the timer, if it is activated. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. */ void TeamUserTimeUserTimer::Deactivate() @@ -845,7 +847,7 @@ TeamUserTimeUserTimer::Deactivate() /*! Checks whether the timer is up, firing an event, if so. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. */ void TeamUserTimeUserTimer::Check() @@ -899,7 +901,7 @@ ThreadTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, uint32 flags, bigtime_t& _oldRemainingTime, bigtime_t& _oldInterval) { InterruptsWriteSequentialLocker locker(sUserTimerLock); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + SpinLocker timeLocker(fThread->time_lock); // get the current time, but only if needed bool nowValid = fThread != NULL; @@ -933,9 +935,13 @@ ThreadTimeUserTimer::Schedule(bigtime_t nextTime, bigtime_t interval, // Get the thread. If it doesn't exist anymore, just don't schedule the // timer anymore. - fThread = Thread::Get(fThreadID); - if (fThread == NULL) + Thread* newThread = Thread::Get(fThreadID); + if (newThread == NULL) { + fThread = NULL; return; + } else if (fThread == NULL) + timeLocker.SetTo(newThread->time_lock, false); + fThread = newThread; fAbsolute = (flags & B_RELATIVE_TIMEOUT) == 0; @@ -963,7 +969,7 @@ ThreadTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, count = acquire_read_seqlock(&sUserTimerLock); if (fThread != NULL) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + SpinLocker timeLocker(fThread->time_lock); _remainingTime = fNextTime - fThread->CPUTime(false); _interval = fInterval; } else { @@ -978,7 +984,7 @@ ThreadTimeUserTimer::GetInfo(bigtime_t& _remainingTime, bigtime_t& _interval, /*! Deactivates the timer, if it is activated. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. */ void ThreadTimeUserTimer::Deactivate() @@ -1005,7 +1011,7 @@ ThreadTimeUserTimer::Deactivate() scheduled, or, when the timer was just set and the thread is already running. Schedules a kernel timer for the remaining time. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. */ void ThreadTimeUserTimer::Start() @@ -1066,7 +1072,7 @@ ThreadTimeUserTimer::Stop() /*! Called when the team's CPU time clock which this timer refers to has been set. - The caller must hold the scheduler lock and \c sUserTimerLock. + The caller must hold \c time_lock and \c sUserTimerLock. \param changedBy The value by which the clock has changed. */ @@ -1358,7 +1364,7 @@ create_timer(clockid_t clockID, int32 timerID, Team* team, Thread* thread, /*! Called when the CPU time clock of the given thread has been set. - The caller must hold the scheduler lock. + The caller must hold \c time_lock. \param thread The thread whose CPU time clock has been set. \param changedBy The value by which the CPU time clock has changed @@ -1377,7 +1383,7 @@ thread_clock_changed(Thread* thread, bigtime_t changedBy) /*! Called when the CPU time clock of the given team has been set. - The caller must hold the scheduler lock. + The caller must hold \c time_lock. \param team The team whose CPU time clock has been set. \param changedBy The value by which the CPU time clock has changed @@ -1477,7 +1483,7 @@ user_timer_get_clock(clockid_t clockID, bigtime_t& _time) case CLOCK_THREAD_CPUTIME_ID: { Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(thread->time_lock); _time = thread->CPUTime(false); return B_OK; } @@ -1485,7 +1491,7 @@ user_timer_get_clock(clockid_t clockID, bigtime_t& _time) case CLOCK_PROCESS_USER_CPUTIME_ID: { Team* team = thread_get_current_thread()->team; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(team->time_lock); _time = team->UserCPUTime(); return B_OK; } @@ -1513,7 +1519,7 @@ user_timer_get_clock(clockid_t clockID, bigtime_t& _time) BReference teamReference(team, true); // get the time - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(team->time_lock); _time = team->CPUTime(false); return B_OK; @@ -1630,7 +1636,7 @@ _user_set_clock(clockid_t clockID, bigtime_t time) case CLOCK_THREAD_CPUTIME_ID: { Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(thread->time_lock); bigtime_t diff = time - thread->CPUTime(false); thread->cpu_clock_offset += diff; @@ -1665,7 +1671,7 @@ _user_set_clock(clockid_t clockID, bigtime_t time) BReference teamReference(team, true); // set the time offset - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(team->time_lock); bigtime_t diff = time - team->CPUTime(false); team->cpu_clock_offset += diff; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 0de57db2b6..266f7d878a 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -26,10 +26,14 @@ scheduler_switch_thread(Thread* fromThread, Thread* toThread) user_debug_thread_unscheduled(fromThread); // stop CPU time based user timers + acquire_spinlock(&fromThread->team->time_lock); + acquire_spinlock(&fromThread->time_lock); if (fromThread->HasActiveCPUTimeUserTimers() || fromThread->team->HasActiveCPUTimeUserTimers()) { user_timer_stop_cpu_timers(fromThread, toThread); } + release_spinlock(&fromThread->time_lock); + release_spinlock(&fromThread->team->time_lock); // update CPU and Thread structures and perform the context switch cpu_ent* cpu = fromThread->cpu; @@ -46,10 +50,14 @@ scheduler_switch_thread(Thread* fromThread, Thread* toThread) // first time the same is done in thread.cpp:common_thread_entry(). // continue CPU time based user timers + acquire_spinlock(&fromThread->team->time_lock); + acquire_spinlock(&fromThread->time_lock); if (fromThread->HasActiveCPUTimeUserTimers() || fromThread->team->HasActiveCPUTimeUserTimers()) { user_timer_continue_cpu_timers(fromThread, cpu->previous_thread); } + release_spinlock(&fromThread->time_lock); + release_spinlock(&fromThread->team->time_lock); // notify the user debugger code if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) @@ -69,6 +77,7 @@ scheduler_update_thread_times(Thread* oldThread, Thread* nextThread) } else { acquire_spinlock(&oldThread->time_lock); oldThread->kernel_time += now - oldThread->last_time; + oldThread->last_time = 0; release_spinlock(&oldThread->time_lock); acquire_spinlock(&nextThread->time_lock); @@ -78,8 +87,11 @@ scheduler_update_thread_times(Thread* oldThread, Thread* nextThread) // If the old thread's team has user time timers, check them now. Team* team = oldThread->team; + + acquire_spinlock(&team->time_lock); if (team->HasActiveUserTimeUserTimers()) user_timer_check_team_user_timers(team); + release_spinlock(&team->time_lock); } diff --git a/src/system/kernel/team.cpp b/src/system/kernel/team.cpp index 906ca0b64e..6b651e58cf 100644 --- a/src/system/kernel/team.cpp +++ b/src/system/kernel/team.cpp @@ -490,6 +490,7 @@ Team::Team(team_id id, bool kernel) // init dead/stopped/continued children condition vars dead_children.condition_variable.Init(&dead_children, "team children"); + B_INITIALIZE_SPINLOCK(&time_lock); B_INITIALIZE_SPINLOCK(&signal_lock); fQueuedSignalsCounter = new(std::nothrow) BKernel::QueuedSignalsCounter( @@ -908,7 +909,7 @@ Team::DeactivateCPUTimeUserTimers() /*! Returns the team's current total CPU time (kernel + user + offset). - The caller must hold the scheduler lock. + The caller must hold \c time_lock. \param ignoreCurrentRun If \c true and the current thread is one team's threads, don't add the time since the last time \c last_time was @@ -931,7 +932,7 @@ Team::CPUTime(bool ignoreCurrentRun) const SpinLocker threadTimeLocker(thread->time_lock); time += thread->kernel_time + thread->user_time; - if (thread->IsRunning()) { + if (thread->last_time != 0) { if (!ignoreCurrentRun || thread != currentThread) time += now - thread->last_time; } @@ -943,7 +944,7 @@ Team::CPUTime(bool ignoreCurrentRun) const /*! Returns the team's current user CPU time. - The caller must hold the scheduler lock. + The caller must hold \c time_lock. \return The team's current user CPU time. */ @@ -959,7 +960,7 @@ Team::UserCPUTime() const SpinLocker threadTimeLocker(thread->time_lock); time += thread->user_time; - if (thread->IsRunning() && !thread->in_kernel) + if (thread->last_time != 0 && !thread->in_kernel) time += now - thread->last_time; } @@ -3093,12 +3094,12 @@ team_shutdown_team(Team* team) team->DeleteUserTimers(false); // deactivate CPU time user timers for the team - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker timeLocker(team->time_lock); if (team->HasActiveCPUTimeUserTimers()) team->DeactivateCPUTimeUserTimers(); - schedulerLocker.Unlock(); + timeLocker.Unlock(); // kill all threads but the main thread team_death_entry deathEntry; diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index fa63e16d01..185da50376 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -696,18 +696,22 @@ common_thread_entry(void* _args) // The thread is new and has been scheduled the first time. // start CPU time based user timers + acquire_spinlock(&thread->team->time_lock); + acquire_spinlock(&thread->time_lock); if (thread->HasActiveCPUTimeUserTimers() || thread->team->HasActiveCPUTimeUserTimers()) { user_timer_continue_cpu_timers(thread, thread->cpu->previous_thread); } + // start tracking time + thread->last_time = system_time(); + release_spinlock(&thread->time_lock); + release_spinlock(&thread->team->time_lock); + // notify the user debugger code if ((thread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) user_debug_thread_scheduled(thread); - // start tracking time - thread->last_time = system_time(); - // unlock the scheduler lock and enable interrupts release_spinlock(&gSchedulerLock); enable_interrupts(); @@ -1991,10 +1995,10 @@ thread_exit(void) // remember how long this thread lasted bigtime_t now = system_time(); - InterruptsSpinLocker threadTimeLocker(thread->time_lock); + InterruptsSpinLocker teamTimeLocker(team->time_lock); + SpinLocker threadTimeLocker(thread->time_lock); thread->kernel_time += now - thread->last_time; thread->last_time = now; - threadTimeLocker.Unlock(); team->dead_threads_kernel_time += thread->kernel_time; team->dead_threads_user_time += thread->user_time; @@ -2009,6 +2013,9 @@ thread_exit(void) if (thread->HasActiveCPUTimeUserTimers()) thread->DeactivateCPUTimeUserTimers(); + threadTimeLocker.Unlock(); + teamTimeLocker.Unlock(); + // put the thread into the kernel team until it dies remove_thread_from_team(team, thread); insert_thread_into_team(kernelTeam, thread); @@ -2334,11 +2341,10 @@ thread_reset_for_exec(void) thread->user_stack_size = 0; // reset signals - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread->ResetSignalsOnExec(); // reset thread CPU time clock + InterruptsSpinLocker timeLocker(thread->time_lock); thread->cpu_clock_offset = -thread->CPUTime(false); } From 03fb2d886830e4dd4b344c56725db59f96733216 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 8 Nov 2013 02:41:26 +0100 Subject: [PATCH 108/273] kernel: Remove gSchedulerLock * Thread::scheduler_lock protects thread state, priority, etc. * sThreadCreationLock protects thread creation and removal and list of threads in team. * Team::signal_lock and Team::time_lock protect list of threads in team as well. * Scheduler uses its own internal locking. --- headers/private/kernel/condition_variable.h | 27 +++--- headers/private/kernel/kscheduler.h | 19 ++-- headers/private/kernel/listeners.h | 2 +- headers/private/kernel/team.h | 2 +- headers/private/kernel/thread.h | 3 + headers/private/kernel/thread_types.h | 10 +-- .../invalidate_on_exit/invalidate_on_exit.cpp | 6 -- src/libs/compat/freebsd_network/Condvar.cpp | 2 +- src/system/kernel/arch/x86/arch_int.cpp | 2 +- src/system/kernel/condition_variable.cpp | 26 +++--- src/system/kernel/debug/system_profiler.cpp | 13 +-- src/system/kernel/debug/user_debugger.cpp | 8 +- src/system/kernel/device_manager/IOCache.cpp | 6 +- src/system/kernel/image.cpp | 13 ++- src/system/kernel/port.cpp | 8 +- src/system/kernel/scheduler/scheduler.cpp | 35 +++++--- .../kernel/scheduler/scheduler_common.h | 2 + src/system/kernel/sem.cpp | 20 ++--- src/system/kernel/signal.cpp | 24 ++--- src/system/kernel/team.cpp | 44 +++++----- src/system/kernel/thread.cpp | 88 ++++++++++--------- 21 files changed, 182 insertions(+), 178 deletions(-) diff --git a/headers/private/kernel/condition_variable.h b/headers/private/kernel/condition_variable.h index 9f4c0366b6..4e91baec22 100644 --- a/headers/private/kernel/condition_variable.h +++ b/headers/private/kernel/condition_variable.h @@ -56,19 +56,13 @@ public: void Publish(const void* object, const char* objectType); - void Unpublish(bool schedulerLocked = false); + void Unpublish(); - inline void NotifyOne(bool schedulerLocked = false, - status_t result = B_OK); - inline void NotifyAll(bool schedulerLocked = false, - status_t result = B_OK); + inline void NotifyOne(status_t result = B_OK); + inline void NotifyAll(status_t result = B_OK); - static void NotifyOne(const void* object, - bool schedulerLocked = false, - status_t result = B_OK); - static void NotifyAll(const void* object, - bool schedulerLocked = false, - status_t result = B_OK); + static void NotifyOne(const void* object, status_t result); + static void NotifyAll(const void* object, status_t result); // (both methods) caller must ensure that // the variable is not unpublished // concurrently @@ -86,8 +80,7 @@ public: void Dump() const; private: - void _Notify(bool all, bool schedulerLocked, - status_t result); + void _Notify(bool all, status_t result); void _NotifyLocked(bool all, status_t result); protected: @@ -124,16 +117,16 @@ ConditionVariableEntry::~ConditionVariableEntry() inline void -ConditionVariable::NotifyOne(bool schedulerLocked, status_t result) +ConditionVariable::NotifyOne(status_t result) { - _Notify(false, schedulerLocked, result); + _Notify(false, result); } inline void -ConditionVariable::NotifyAll(bool schedulerLocked, status_t result) +ConditionVariable::NotifyAll(status_t result) { - _Notify(true, schedulerLocked, result); + _Notify(true, result); } diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index c1eeb39c1d..bcb5fd6e0d 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -24,15 +24,13 @@ typedef enum scheduler_mode { SCHEDULER_MODE_COUNT } scheduler_mode; -extern spinlock gSchedulerLock; - #ifdef __cplusplus extern "C" { #endif /*! Enqueues the thread in the ready-to-run queue. - The caller must hold the scheduler lock (with disabled interrupts). + The caller must hold the enqueued thread \c scheduler_lock. */ void scheduler_enqueue_in_run_queue(Thread* thread); @@ -42,15 +40,14 @@ void scheduler_enqueue_in_run_queue(Thread* thread); If it's the same thread, the thread will just continue to run. In either case, unless the thread is dead or is sleeping/waiting indefinitely, the function will eventually return. - The caller must hold the scheduler lock (with disabled interrupts). + The caller must hold the current thread \c scheduler_lock. */ void scheduler_reschedule(void); /*! Sets the given thread's priority. The thread may be running or may be in the ready-to-run queue. - The caller must hold the scheduler lock (with disabled interrupts). */ -void scheduler_set_thread_priority(Thread* thread, int32 priority); +int32 scheduler_set_thread_priority(Thread* thread, int32 priority); /*! Called when the Thread structure is first created. Per-thread housekeeping resources can be allocated. @@ -61,7 +58,6 @@ status_t scheduler_on_thread_create(Thread* thread, bool idleThread); /*! Called when a Thread structure is initialized and made ready for use. The per-thread housekeeping data structures are reset, if needed. - The caller must hold the scheduler lock (with disabled interrupts). */ void scheduler_on_thread_init(Thread* thread); @@ -75,8 +71,6 @@ void scheduler_on_thread_destroy(Thread* thread); /*! Called in the early boot process to start thread scheduling on the current CPU. The function is called once for each CPU. - Interrupts must be disabled, but the caller must not hold the scheduler - lock. */ void scheduler_start(void); @@ -122,11 +116,14 @@ scheduler_reschedule_if_necessary() { if (are_interrupts_enabled()) { cpu_status state = disable_interrupts(); - acquire_spinlock(&gSchedulerLock); + + Thread* thread = get_cpu_struct()->running_thread; + acquire_spinlock(&thread->scheduler_lock); scheduler_reschedule_if_necessary_locked(); - release_spinlock(&gSchedulerLock); + release_spinlock(&thread->scheduler_lock); + restore_interrupts(state); } } diff --git a/headers/private/kernel/listeners.h b/headers/private/kernel/listeners.h index 64bb94d3d1..6406435325 100644 --- a/headers/private/kernel/listeners.h +++ b/headers/private/kernel/listeners.h @@ -34,7 +34,7 @@ struct SchedulerListener : DoublyLinkedListLinkImpl { typedef DoublyLinkedList SchedulerListenerList; extern SchedulerListenerList gSchedulerListeners; - // guarded by the thread spinlock +extern spinlock gSchedulerListenersLock; template diff --git a/headers/private/kernel/team.h b/headers/private/kernel/team.h index ce09ea470a..2fb555e493 100644 --- a/headers/private/kernel/team.h +++ b/headers/private/kernel/team.h @@ -46,7 +46,7 @@ thread_id load_image_etc(int32 argCount, const char* const* args, const char* const* env, int32 priority, team_id parentID, uint32 flags); void team_set_job_control_state(Team* team, job_control_state newState, - Signal* signal, bool threadsLocked); + Signal* signal); void team_set_controlling_tty(int32 index); int32 team_get_controlling_tty(); status_t team_set_foreground_process_group(int32 ttyIndex, pid_t processGroup); diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 9cc12499c2..72d123b503 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -69,6 +69,9 @@ public: using BKernel::ThreadCreationAttributes; +extern spinlock gThreadCreationLock; + + #ifdef __cplusplus extern "C" { #endif diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index c0448213e3..6212deb48d 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -243,10 +243,10 @@ struct Team : TeamThreadIteratorEntry, KernelReferenceable, struct job_control_entry* job_control_entry; VMAddressSpace *address_space; - Thread *main_thread; // protected by fLock and the scheduler - // lock (and the thread's lock), immutable + Thread *main_thread; // protected by fLock, immutable // after first set - Thread *thread_list; // protected by fLock and the scheduler lock + Thread *thread_list; // protected by fLock, signal_lock and + // gThreadCreationLock struct team_loading_info *loading_info; // protected by fLock struct list image_list; // protected by sImageMutex struct list watcher_list; @@ -270,8 +270,7 @@ struct Team : TeamThreadIteratorEntry, KernelReferenceable, bigtime_t cpu_clock_offset; spinlock time_lock; - // user group information; protected by fLock, the *_uid/*_gid fields also - // by the scheduler lock + // user group information; protected by fLock uid_t saved_set_uid; uid_t real_uid; uid_t effective_uid; @@ -430,6 +429,7 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, struct cpu_ent *previous_cpu; // protected by scheduler lock int32 pinned_to_cpu; // only accessed by this thread or in the // scheduler, when thread is not running + spinlock scheduler_lock; sigset_t sig_block_mask; // protected by team->signal_lock, // only modified by the thread itself diff --git a/src/add-ons/kernel/debugger/invalidate_on_exit/invalidate_on_exit.cpp b/src/add-ons/kernel/debugger/invalidate_on_exit/invalidate_on_exit.cpp index 3ff72b3842..abcea1c286 100644 --- a/src/add-ons/kernel/debugger/invalidate_on_exit/invalidate_on_exit.cpp +++ b/src/add-ons/kernel/debugger/invalidate_on_exit/invalidate_on_exit.cpp @@ -39,12 +39,6 @@ invalidate_loop(void *data) static void exit_debugger() { - // If someone holds the scheduler lock at this point, release_sem_etc() - // will block forever. So avoid that. - if (!try_acquire_spinlock(&gSchedulerLock)) - return; - release_spinlock(&gSchedulerLock); - release_sem_etc(sRequestSem, 1, B_DO_NOT_RESCHEDULE); } diff --git a/src/libs/compat/freebsd_network/Condvar.cpp b/src/libs/compat/freebsd_network/Condvar.cpp index 74499ef0cb..9cb5be1dcd 100644 --- a/src/libs/compat/freebsd_network/Condvar.cpp +++ b/src/libs/compat/freebsd_network/Condvar.cpp @@ -77,5 +77,5 @@ publishedConditionTimedWait(const void* waitChannel, const int timeout) void publishedConditionNotifyAll(const void* waitChannel) { - ConditionVariable::NotifyAll(waitChannel); + ConditionVariable::NotifyAll(waitChannel, B_OK); } diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 1fabe85b1b..3591f909e8 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -231,7 +231,7 @@ x86_hardware_interrupt(struct iframe* frame) cpu_status state = disable_interrupts(); if (thread->cpu->invoke_scheduler) { - SpinLocker schedulerLocker(gSchedulerLock); + SpinLocker schedulerLocker(thread->scheduler_lock); scheduler_reschedule(); schedulerLocker.Unlock(); restore_interrupts(state); diff --git a/src/system/kernel/condition_variable.cpp b/src/system/kernel/condition_variable.cpp index 7f6152ebce..85c7a3f87b 100644 --- a/src/system/kernel/condition_variable.cpp +++ b/src/system/kernel/condition_variable.cpp @@ -217,13 +217,11 @@ ConditionVariable::Publish(const void* object, const char* objectType) void -ConditionVariable::Unpublish(bool schedulerLocked) +ConditionVariable::Unpublish() { ASSERT(fObject != NULL); - InterruptsLocker _; - SpinLocker schedulerLocker(schedulerLocked ? NULL : &gSchedulerLock); - SpinLocker locker(sConditionVariablesLock); + InterruptsSpinLocker locker(sConditionVariablesLock); #if KDEBUG ConditionVariable* variable = sConditionVariableHash.Lookup(fObject); @@ -259,8 +257,7 @@ ConditionVariable::Wait(uint32 flags, bigtime_t timeout) /*static*/ void -ConditionVariable::NotifyOne(const void* object, bool schedulerLocked, - status_t result) +ConditionVariable::NotifyOne(const void* object, status_t result) { InterruptsSpinLocker locker(sConditionVariablesLock); ConditionVariable* variable = sConditionVariableHash.Lookup(object); @@ -268,13 +265,12 @@ ConditionVariable::NotifyOne(const void* object, bool schedulerLocked, if (variable == NULL) return; - variable->NotifyOne(schedulerLocked, result); + variable->NotifyOne(result); } /*static*/ void -ConditionVariable::NotifyAll(const void* object, bool schedulerLocked, - status_t result) +ConditionVariable::NotifyAll(const void* object, status_t result) { InterruptsSpinLocker locker(sConditionVariablesLock); ConditionVariable* variable = sConditionVariableHash.Lookup(object); @@ -282,7 +278,7 @@ ConditionVariable::NotifyAll(const void* object, bool schedulerLocked, if (variable == NULL) return; - variable->NotifyAll(schedulerLocked, result); + variable->NotifyAll(result); } @@ -318,11 +314,9 @@ ConditionVariable::Dump() const void -ConditionVariable::_Notify(bool all, bool schedulerLocked, status_t result) +ConditionVariable::_Notify(bool all, status_t result) { - InterruptsLocker _; - SpinLocker schedulerLocker(schedulerLocked ? NULL : &gSchedulerLock); - SpinLocker locker(sConditionVariablesLock); + InterruptsSpinLocker locker(sConditionVariablesLock); if (!fEntries.IsEmpty()) { if (result > B_OK) { @@ -348,8 +342,10 @@ ConditionVariable::_NotifyLocked(bool all, status_t result) if (entry->fWaitStatus <= 0) continue; - if (entry->fWaitStatus == STATUS_WAITING) + if (entry->fWaitStatus == STATUS_WAITING) { + SpinLocker _(entry->fThread->scheduler_lock); thread_unblock_locked(entry->fThread, result); + } entry->fWaitStatus = result; diff --git a/src/system/kernel/debug/system_profiler.cpp b/src/system/kernel/debug/system_profiler.cpp index d0d3e22016..5b341b2254 100644 --- a/src/system/kernel/debug/system_profiler.cpp +++ b/src/system/kernel/debug/system_profiler.cpp @@ -220,6 +220,7 @@ SystemProfiler::_MaybeNotifyProfilerThreadLocked() int cpu = smp_get_current_cpu(); fReentered[cpu] = true; + InterruptsSpinLocker _(fWaitingProfilerThread->scheduler_lock); thread_unblock_locked(fWaitingProfilerThread, B_OK); fWaitingProfilerThread = NULL; @@ -234,7 +235,6 @@ SystemProfiler::_MaybeNotifyProfilerThread() if (fWaitingProfilerThread == NULL) return; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); SpinLocker locker(fLock); _MaybeNotifyProfilerThreadLocked(); @@ -305,10 +305,8 @@ SystemProfiler::~SystemProfiler() locker.Unlock(); // stop scheduler listening - if (fSchedulerNotificationsRequested) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + if (fSchedulerNotificationsRequested) scheduler_remove_listener(this); - } // stop wait object listening if (fWaitObjectNotificationsRequested) { @@ -498,8 +496,6 @@ SystemProfiler::Init() fThreadNotificationsEnabled = true; } - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - fProfilingActive = true; // start scheduler and wait object listening @@ -521,8 +517,6 @@ SystemProfiler::Init() } } - schedulerLocker.Unlock(); - // I/O scheduling if ((fFlags & B_SYSTEM_PROFILER_IO_SCHEDULING_EVENTS) != 0) { IOSchedulerRoster* roster = IOSchedulerRoster::Default(); @@ -571,12 +565,9 @@ SystemProfiler::NextBuffer(size_t bytesRead, uint64* _droppedEvents) Thread* thread = thread_get_current_thread(); fWaitingProfilerThread = thread; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - thread_prepare_to_block(thread, B_CAN_INTERRUPT, THREAD_BLOCK_TYPE_OTHER, "system profiler buffer"); - schedulerLocker.Unlock(); locker.Unlock(); status_t error = thread_block_with_timeout(B_RELATIVE_TIMEOUT, 1000000); diff --git a/src/system/kernel/debug/user_debugger.cpp b/src/system/kernel/debug/user_debugger.cpp index a3f1915878..6999556c04 100644 --- a/src/system/kernel/debug/user_debugger.cpp +++ b/src/system/kernel/debug/user_debugger.cpp @@ -440,7 +440,7 @@ finish_debugger_change(Team* team) ConditionVariable* condition = team->debug_info.debugger_changed_condition; team->debug_info.debugger_changed_condition = NULL; - condition->NotifyAll(false); + condition->NotifyAll(); } @@ -2901,7 +2901,7 @@ _user_debug_thread(thread_id threadID) // resume/interrupt the thread, if necessary threadDebugInfoLocker.Unlock(); - SpinLocker schedulerLocker(gSchedulerLock); + SpinLocker schedulerLocker(thread->scheduler_lock); switch (thread->state) { case B_THREAD_SUSPENDED: @@ -2916,6 +2916,10 @@ _user_debug_thread(thread_id threadID) // about to acquire a semaphore (before // thread_prepare_to_block()), we won't interrupt it. // Maybe we should rather send a signal (SIGTRAP). + schedulerLocker.Unlock(); + + schedulerLocker.SetTo(thread_get_current_thread()->scheduler_lock, + false); scheduler_reschedule_if_necessary_locked(); break; } diff --git a/src/system/kernel/device_manager/IOCache.cpp b/src/system/kernel/device_manager/IOCache.cpp index a65e515d28..7cd834b8a5 100644 --- a/src/system/kernel/device_manager/IOCache.cpp +++ b/src/system/kernel/device_manager/IOCache.cpp @@ -205,10 +205,10 @@ IOCache::OperationCompleted(IOOperation* operation, status_t status, { if (status == B_OK) { // always fail in case of partial transfers - ((Operation*)operation)->finishedCondition.NotifyAll(false, + ((Operation*)operation)->finishedCondition.NotifyAll( transferredBytes == operation->Length() ? B_OK : B_ERROR); } else - ((Operation*)operation)->finishedCondition.NotifyAll(false, status); + ((Operation*)operation)->finishedCondition.NotifyAll(status); } @@ -498,7 +498,7 @@ IOCache::_DoOperation(Operation& operation) status_t error = fIOCallback(fIOCallbackData, &operation); if (error != B_OK) { - operation.finishedCondition.NotifyAll(false, error); + operation.finishedCondition.NotifyAll(error); // removes the entry from the variable return error; } diff --git a/src/system/kernel/image.cpp b/src/system/kernel/image.cpp index e76cc60118..55ac115dff 100644 --- a/src/system/kernel/image.cpp +++ b/src/system/kernel/image.cpp @@ -370,15 +370,20 @@ notify_loading_app(status_t result, bool suspend) // we're done with the team stuff, get the scheduler lock instead teamLocker.Unlock(); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + Thread* thread = loadingInfo->thread; + InterruptsSpinLocker schedulerLocker(thread->scheduler_lock); // wake up the waiting thread - if (loadingInfo->thread->state == B_THREAD_SUSPENDED) - scheduler_enqueue_in_run_queue(loadingInfo->thread); + if (thread->state == B_THREAD_SUSPENDED) + scheduler_enqueue_in_run_queue(thread); + schedulerLocker.Unlock(); // suspend ourselves, if desired if (suspend) { - thread_get_current_thread()->next_state = B_THREAD_SUSPENDED; + Thread* thread = thread_get_current_thread(); + InterruptsSpinLocker schedulerLocker(thread->scheduler_lock); + + thread->next_state = B_THREAD_SUSPENDED; scheduler_reschedule(); } } diff --git a/src/system/kernel/port.cpp b/src/system/kernel/port.cpp index e4ba96c7ae..d5b6700db0 100644 --- a/src/system/kernel/port.cpp +++ b/src/system/kernel/port.cpp @@ -684,8 +684,8 @@ uninit_port_locked(Port* port) // Release the threads that were blocking on this port. // read_port() will see the B_BAD_PORT_ID return value, and act accordingly - port->read_condition.NotifyAll(false, B_BAD_PORT_ID); - port->write_condition.NotifyAll(false, B_BAD_PORT_ID); + port->read_condition.NotifyAll(B_BAD_PORT_ID); + port->write_condition.NotifyAll(B_BAD_PORT_ID); sNotificationService.Notify(PORT_REMOVED, port->id); } @@ -891,8 +891,8 @@ close_port(port_id id) notify_port_select_events(port, B_EVENT_INVALID); port->select_infos = NULL; - port->read_condition.NotifyAll(false, B_BAD_PORT_ID); - port->write_condition.NotifyAll(false, B_BAD_PORT_ID); + port->read_condition.NotifyAll(B_BAD_PORT_ID); + port->write_condition.NotifyAll(B_BAD_PORT_ID); return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index e113e25dde..6f7a2bf050 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -45,10 +45,11 @@ #endif -spinlock gSchedulerLock = B_SPINLOCK_INITIALIZER; SchedulerListenerList gSchedulerListeners; +spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -bool sSchedulerEnabled; +static spinlock sSchedulerInternalLock; +static bool sSchedulerEnabled; const bigtime_t kThreadQuantum = 1000; const bigtime_t kMinThreadQuantum = 3000; @@ -1129,6 +1130,8 @@ enqueue(Thread* thread, bool newOne) void scheduler_enqueue_in_run_queue(Thread *thread) { + InterruptsSpinLocker _(sSchedulerInternalLock); + TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); enqueue(thread, true); @@ -1156,17 +1159,19 @@ put_back(Thread* thread) /*! Sets the priority of a thread. - Note: thread lock must be held when entering this function */ -void +int32 scheduler_set_thread_priority(Thread *thread, int32 priority) { + InterruptsSpinLocker _(sSchedulerInternalLock); + if (priority == thread->priority) - return; + return thread->priority; + + int32 oldPriority = thread->priority; TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", - thread->id, priority, thread->priority, - get_effective_priority(thread)); + thread->id, priority, oldPriority, get_effective_priority(thread)); if (thread->state != B_THREAD_READY) { cancel_penalty(thread); @@ -1174,7 +1179,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) if (thread->state == B_THREAD_RUNNING) update_priority_heaps(thread->cpu->cpu_num, priority); - return; + return oldPriority; } // The thread is in the run queue. We need to remove it and re-insert it at @@ -1194,8 +1199,9 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) // set priority and re-insert cancel_penalty(thread); thread->priority = priority; + enqueue(thread, true); - scheduler_enqueue_in_run_queue(thread); + return oldPriority; } @@ -1403,6 +1409,8 @@ update_cpu_performance(Thread* thread, int32 thisCore) static void _scheduler_reschedule(void) { + InterruptsSpinLocker internalLocker(sSchedulerInternalLock); + Thread* oldThread = thread_get_current_thread(); int32 thisCPU = smp_get_current_cpu(); @@ -1473,6 +1481,8 @@ _scheduler_reschedule(void) nextThread = dequeue_thread(thisCPU); if (!nextThread) panic("reschedule(): run queues are empty!\n"); + if (nextThread != oldThread) + acquire_spinlock(&nextThread->scheduler_lock); TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, nextThread->id); @@ -1513,6 +1523,7 @@ _scheduler_reschedule(void) } else nextThread->scheduler_data->quantum_start = system_time(); + internalLocker.Unlock(); if (nextThread != oldThread) scheduler_switch_thread(oldThread, nextThread); } @@ -1566,7 +1577,7 @@ scheduler_on_thread_destroy(Thread* thread) void scheduler_start(void) { - SpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); _scheduler_reschedule(); } @@ -1583,7 +1594,7 @@ scheduler_set_operation_mode(scheduler_mode mode) const char* modeNames[] = { "performance", "power saving" }; dprintf("scheduler: switching to %s mode\n", modeNames[mode]); - InterruptsSpinLocker _(gSchedulerLock); + InterruptsSpinLocker _(sSchedulerInternalLock); sSchedulerMode = mode; switch (mode) { @@ -1877,6 +1888,7 @@ SchedulerListener::~SchedulerListener() void scheduler_add_listener(struct SchedulerListener* listener) { + InterruptsSpinLocker _(gSchedulerListenersLock); gSchedulerListeners.Add(listener); } @@ -1886,6 +1898,7 @@ scheduler_add_listener(struct SchedulerListener* listener) void scheduler_remove_listener(struct SchedulerListener* listener) { + InterruptsSpinLocker _(gSchedulerListenersLock); gSchedulerListeners.Remove(listener); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 266f7d878a..a105eb8a52 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -45,6 +45,8 @@ scheduler_switch_thread(Thread* fromThread, Thread* toThread) arch_thread_set_current_thread(toThread); arch_thread_context_switch(fromThread, toThread); + release_spinlock(&fromThread->cpu->previous_thread->scheduler_lock); + // The use of fromThread below looks weird, but is correct. fromThread had // been unscheduled earlier, but is back now. For a thread scheduled the // first time the same is done in thread.cpp:common_thread_entry(). diff --git a/src/system/kernel/sem.cpp b/src/system/kernel/sem.cpp index 7980ed29e2..b947f2080e 100644 --- a/src/system/kernel/sem.cpp +++ b/src/system/kernel/sem.cpp @@ -393,7 +393,7 @@ delete_sem_internal(sem_id id, bool checkPermission) char* name; uninit_sem_locked(sSems[slot], &name); - SpinLocker schedulerLocker(gSchedulerLock); + SpinLocker schedulerLocker(thread_get_current_thread()->scheduler_lock); scheduler_reschedule_if_necessary_locked(); schedulerLocker.Unlock(); @@ -644,9 +644,8 @@ remove_thread_from_sem(queued_thread *entry, struct sem_entry *sem) // for that time, so the blocking state of threads won't change (due to // interruption or timeout). We need that lock anyway when unblocking a // thread. - SpinLocker schedulerLocker(gSchedulerLock); - while ((entry = sem->queue.Head()) != NULL) { + SpinLocker schedulerLocker(entry->thread->scheduler_lock); if (thread_is_blocked(entry->thread)) { // The thread is still waiting. If its count is satisfied, unblock // it. Otherwise we can't unblock any other thread. @@ -665,8 +664,6 @@ remove_thread_from_sem(queued_thread *entry, struct sem_entry *sem) entry->queued = false; } - schedulerLocker.Unlock(); - // select notification, if the semaphore is now acquirable if (sem->u.used.count > 0) notify_sem_select_events(sem, B_EVENT_ACQUIRE_SEMAPHORE); @@ -823,7 +820,7 @@ switch_sem_etc(sem_id semToBeReleased, sem_id id, int32 count, // do a quick check to see if the thread has any pending signals // this should catch most of the cases where the thread had a signal - SpinLocker schedulerLocker(gSchedulerLock); + SpinLocker schedulerLocker(thread->scheduler_lock); if (thread_is_interrupted(thread, flags)) { schedulerLocker.Unlock(); sSems[slot].u.used.count += count; @@ -832,6 +829,8 @@ switch_sem_etc(sem_id semToBeReleased, sem_id id, int32 count, goto err; } + schedulerLocker.Unlock(); + if ((flags & (B_RELATIVE_TIMEOUT | B_ABSOLUTE_TIMEOUT)) == 0) timeout = B_INFINITE_TIMEOUT; @@ -843,7 +842,6 @@ switch_sem_etc(sem_id semToBeReleased, sem_id id, int32 count, thread_prepare_to_block(thread, flags, THREAD_BLOCK_TYPE_SEMAPHORE, (void*)(addr_t)id); - schedulerLocker.Unlock(); RELEASE_SEM_LOCK(sSems[slot]); // release the other semaphore, if any @@ -968,8 +966,6 @@ release_sem_etc(sem_id id, int32 count, uint32 flags) // Grab the scheduler lock, so thread_is_blocked() is reliable (due to // possible interruptions or timeouts, it wouldn't be otherwise). - SpinLocker schedulerLocker(gSchedulerLock); - while (count > 0) { queued_thread* entry = sSems[slot].queue.Head(); if (entry == NULL) { @@ -980,6 +976,7 @@ release_sem_etc(sem_id id, int32 count, uint32 flags) break; } + SpinLocker schedulerLock(entry->thread->scheduler_lock); if (thread_is_blocked(entry->thread)) { // The thread is still waiting. If its count is satisfied, // unblock it. Otherwise we can't unblock any other thread. @@ -1005,8 +1002,6 @@ release_sem_etc(sem_id id, int32 count, uint32 flags) entry->queued = false; } - schedulerLocker.Unlock(); - if (sSems[slot].u.used.count > 0) notify_sem_select_events(&sSems[slot], B_EVENT_ACQUIRE_SEMAPHORE); @@ -1014,7 +1009,8 @@ release_sem_etc(sem_id id, int32 count, uint32 flags) // been told not to. if ((flags & B_DO_NOT_RESCHEDULE) == 0) { semLocker.Unlock(); - schedulerLocker.Lock(); + + SpinLocker _(thread_get_current_thread()->scheduler_lock); scheduler_reschedule_if_necessary_locked(); } diff --git a/src/system/kernel/signal.cpp b/src/system/kernel/signal.cpp index 7907dfa9b4..5377903308 100644 --- a/src/system/kernel/signal.cpp +++ b/src/system/kernel/signal.cpp @@ -1064,7 +1064,7 @@ handle_signals(Thread* thread) team->LockTeamAndParent(false); team_set_job_control_state(team, - JOB_CONTROL_STATE_CONTINUED, signal, false); + JOB_CONTROL_STATE_CONTINUED, signal); team->UnlockTeamAndParent(); @@ -1099,7 +1099,7 @@ handle_signals(Thread* thread) team->LockTeamAndParent(false); team_set_job_control_state(team, - JOB_CONTROL_STATE_STOPPED, signal, false); + JOB_CONTROL_STATE_STOPPED, signal); // send a SIGCHLD to the parent (if it does have // SA_NOCLDSTOP defined) @@ -1128,7 +1128,7 @@ handle_signals(Thread* thread) locker.Unlock(); if (!resume) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker _(thread->scheduler_lock); thread->next_state = B_THREAD_SUSPENDED; scheduler_reschedule(); } @@ -1377,7 +1377,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, if (thread->team == team_get_kernel_team()) { // Signals to kernel threads will only wake them up - SpinLocker _(gSchedulerLock); + SpinLocker _(thread->scheduler_lock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); return B_OK; @@ -1401,7 +1401,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, mainThread->AddPendingSignal(SIGKILLTHR); // wake up main thread - SpinLocker locker(gSchedulerLock); + SpinLocker locker(mainThread->scheduler_lock); if (mainThread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(mainThread); else @@ -1416,7 +1416,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, case SIGKILLTHR: { // Wake up suspended threads and interrupt waiting ones - SpinLocker locker(gSchedulerLock); + SpinLocker locker(thread->scheduler_lock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); else @@ -1427,7 +1427,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, case SIGNAL_CONTINUE_THREAD: { // wake up thread, and interrupt its current syscall - SpinLocker locker(gSchedulerLock); + SpinLocker locker(thread->scheduler_lock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); @@ -1438,7 +1438,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, { // Wake up thread if it was suspended, otherwise interrupt it, if // the signal isn't blocked. - SpinLocker locker(gSchedulerLock); + SpinLocker locker(thread->scheduler_lock); if (thread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(thread); else if ((SIGNAL_TO_MASK(SIGCONT) & ~thread->sig_block_mask) != 0) @@ -1455,7 +1455,7 @@ send_signal_to_thread_locked(Thread* thread, uint32 signalNumber, & (~thread->sig_block_mask | SIGNAL_TO_MASK(SIGCHLD))) != 0) { // Interrupt thread if it was waiting - SpinLocker locker(gSchedulerLock); + SpinLocker locker(thread->scheduler_lock); thread_interrupt(thread, false); } break; @@ -1605,7 +1605,7 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, mainThread->AddPendingSignal(SIGKILLTHR); // wake up main thread - SpinLocker _(gSchedulerLock); + SpinLocker _(mainThread->scheduler_lock); if (mainThread->state == B_THREAD_SUSPENDED) scheduler_enqueue_in_run_queue(mainThread); else @@ -1619,7 +1619,7 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, // don't block the signal. for (Thread* thread = team->thread_list; thread != NULL; thread = thread->team_next) { - SpinLocker _(gSchedulerLock); + SpinLocker _(thread->scheduler_lock); if (thread->state == B_THREAD_SUSPENDED) { scheduler_enqueue_in_run_queue(thread); } else if ((SIGNAL_TO_MASK(SIGCONT) & ~thread->sig_block_mask) @@ -1662,7 +1662,7 @@ send_signal_to_team_locked(Team* team, uint32 signalNumber, Signal* signal, sigset_t nonBlocked = ~thread->sig_block_mask | SIGNAL_TO_MASK(SIGCHLD); if ((thread->AllPendingSignals() & nonBlocked) != 0) { - SpinLocker _(gSchedulerLock); + SpinLocker _(thread->scheduler_lock); thread_interrupt(thread, false); } } diff --git a/src/system/kernel/team.cpp b/src/system/kernel/team.cpp index 6b651e58cf..175645a15c 100644 --- a/src/system/kernel/team.cpp +++ b/src/system/kernel/team.cpp @@ -1794,11 +1794,13 @@ load_image_internal(char**& _flatArgs, size_t flatArgsSize, int32 argCount, // wait for the loader of the new team to finish its work if ((flags & B_WAIT_TILL_LOADED) != 0) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + if (mainThread != NULL) { + InterruptsSpinLocker schedulerLocker(mainThread->scheduler_lock); - // resume the team's main thread - if (mainThread != NULL && mainThread->state == B_THREAD_SUSPENDED) - scheduler_enqueue_in_run_queue(mainThread); + // resume the team's main thread + if (mainThread->state == B_THREAD_SUSPENDED) + scheduler_enqueue_in_run_queue(mainThread); + } // Now suspend ourselves until loading is finished. We will be woken // either by the thread, when it finished or aborted loading, or when @@ -1806,12 +1808,13 @@ load_image_internal(char**& _flatArgs, size_t flatArgsSize, int32 argCount, // setting `loadingInfo.done' is responsible for removing the info from // the team structure. while (!loadingInfo.done) { - thread_get_current_thread()->next_state = B_THREAD_SUSPENDED; + Thread* thread = thread_get_current_thread(); + + InterruptsSpinLocker schedulerLocker(thread->scheduler_lock); + thread->next_state = B_THREAD_SUSPENDED; scheduler_reschedule(); } - schedulerLocker.Unlock(); - if (loadingInfo.result < B_OK) return loadingInfo.result; } @@ -2444,7 +2447,7 @@ wait_for_child(pid_t child, uint32 flags, siginfo_t& _info) } else { // The child is well. Reset its job control state. team_set_job_control_state(entry->team, - JOB_CONTROL_STATE_NONE, NULL, false); + JOB_CONTROL_STATE_NONE, NULL); } } } @@ -2531,14 +2534,16 @@ wait_for_child(pid_t child, uint32 flags, siginfo_t& _info) // If SIGCHLD is blocked, we shall clear pending SIGCHLDs, if no other child // status is available. TeamLocker teamLocker(team); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker signalLocker(team->signal_lock); + SpinLocker threadCreationLocker(gThreadCreationLock); if (is_team_signal_blocked(team, SIGCHLD)) { if (get_job_control_entry(team, child, flags) == NULL) team->RemovePendingSignals(SIGNAL_TO_MASK(SIGCHLD)); } - schedulerLocker.Unlock(); + threadCreationLocker.Unlock(); + signalLocker.Unlock(); teamLocker.Unlock(); // When the team is dead, the main thread continues to live in the kernel @@ -2925,12 +2930,12 @@ team_set_foreground_process_group(int32 ttyIndex, pid_t processGroupID) if (session->foreground_group != -1 && session->foreground_group != team->group_id && team->SignalActionFor(SIGTTOU).sa_handler != SIG_IGN) { - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker signalLocker(team->signal_lock); if (!is_team_signal_blocked(team, SIGTTOU)) { pid_t groupID = team->group_id; - schedulerLocker.Unlock(); + signalLocker.Unlock(); sessionLocker.Unlock(); teamLocker.Unlock(); @@ -3170,7 +3175,7 @@ team_delete_team(Team* team, port_id debuggerPort) loadingInfo->result = B_ERROR; loadingInfo->done = true; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker _(loadingInfo->thread->scheduler_lock); // wake up the waiting thread if (loadingInfo->thread->state == B_THREAD_SUSPENDED) @@ -3256,8 +3261,7 @@ team_get_address_space(team_id id, VMAddressSpace** _addressSpace) /*! Sets the team's job control state. The caller must hold the parent team's lock. Interrupts are allowed to be - enabled or disabled. In the latter case the scheduler lock may be held as - well. + enabled or disabled. \a team The team whose job control state shall be set. \a newState The new state to be set. \a signal The signal the new state was caused by. Can \c NULL, if none. Then @@ -3266,11 +3270,10 @@ team_get_address_space(team_id id, VMAddressSpace** _addressSpace) \c JOB_CONTROL_STATE_NONE: - \c signal: The number of the signal causing the state change. - \c signaling_user: The real UID of the user sending the signal. - \a schedulerLocked indicates whether the scheduler lock is being held, too. */ void team_set_job_control_state(Team* team, job_control_state newState, - Signal* signal, bool schedulerLocked) + Signal* signal) { if (team == NULL || team->job_control_entry == NULL) return; @@ -3326,8 +3329,7 @@ team_set_job_control_state(Team* team, job_control_state newState, if (childList != NULL) { childList->entries.Add(entry); - team->parent->dead_children.condition_variable.NotifyAll( - schedulerLocked); + team->parent->dead_children.condition_variable.NotifyAll(); } } @@ -4087,7 +4089,7 @@ _user_setpgid(pid_t processID, pid_t groupID) // Changing the process group might have changed the situation for a // parent waiting in wait_for_child(). Hence we notify it. - team->parent->dead_children.condition_variable.NotifyAll(false); + team->parent->dead_children.condition_variable.NotifyAll(); return group->id; } @@ -4129,7 +4131,7 @@ _user_setsid(void) // Changing the process group might have changed the situation for a // parent waiting in wait_for_child(). Hence we notify it. - team->parent->dead_children.condition_variable.NotifyAll(false); + team->parent->dead_children.condition_variable.NotifyAll(); return group->id; } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 185da50376..7b1cdcbcde 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -81,6 +81,8 @@ static thread_id sNextThreadID = 2; static int32 sMaxThreads = 4096; static int32 sUsedThreads = 0; +spinlock gThreadCreationLock = B_SPINLOCK_INITIALIZER; + struct UndertakerEntry : DoublyLinkedListLinkImpl { Thread* thread; @@ -206,6 +208,7 @@ Thread::Thread(const char* name, thread_id threadID, struct cpu_ent* cpu) mutex_init_etc(&fLock, lockName, MUTEX_FLAG_CLONE_NAME); B_INITIALIZE_SPINLOCK(&time_lock); + B_INITIALIZE_SPINLOCK(&scheduler_lock); B_INITIALIZE_RW_SPINLOCK(&team_lock); // init name @@ -713,7 +716,8 @@ common_thread_entry(void* _args) user_debug_thread_scheduled(thread); // unlock the scheduler lock and enable interrupts - release_spinlock(&gSchedulerLock); + release_spinlock(&thread->cpu->previous_thread->scheduler_lock); + release_spinlock(&thread->scheduler_lock); enable_interrupts(); // call the kernel function, if any @@ -1020,7 +1024,8 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel) // for our own use (and threadReference remains armed). ThreadLocker threadLocker(thread); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + + InterruptsSpinLocker threadCreationLocker(gThreadCreationLock); SpinLocker threadHashLocker(sThreadHashLock); // check the thread limit @@ -1028,7 +1033,7 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel) // Clean up the user_thread structure. It's a bit unfortunate that the // Thread destructor cannot do that, so we have to do that explicitly. threadHashLocker.Unlock(); - schedulerLocker.Unlock(); + threadCreationLocker.Unlock(); user_thread* userThread = thread->user_thread; thread->user_thread = NULL; @@ -1044,6 +1049,7 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel) // make thread visible in global hash/list thread->visible = true; sUsedThreads++; + scheduler_on_thread_init(thread); thread->AcquireReference(); @@ -1060,11 +1066,16 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel) } } - // insert thread into team - insert_thread_into_team(team, thread); + { + SpinLocker signalLocker(team->signal_lock); + SpinLocker timeLocker(team->time_lock); + + // insert thread into team + insert_thread_into_team(team, thread); + } threadHashLocker.Unlock(); - schedulerLocker.Unlock(); + threadCreationLocker.Unlock(); threadLocker.Unlock(); teamLocker.Unlock(); @@ -1106,13 +1117,17 @@ undertaker(void* /*args*/) Team* kernelTeam = team_get_kernel_team(); TeamLocker kernelTeamLocker(kernelTeam); thread->Lock(); - InterruptsSpinLocker signalLocker(kernelTeam->signal_lock); - SpinLocker schedulerLocker(gSchedulerLock); + + InterruptsSpinLocker threadCreationLocker(gThreadCreationLock); + SpinLocker signalLocker(kernelTeam->signal_lock); + SpinLocker timeLocker(kernelTeam->time_lock); remove_thread_from_team(kernelTeam, thread); - schedulerLocker.Unlock(); + timeLocker.Unlock(); signalLocker.Unlock(); + threadCreationLocker.Unlock(); + kernelTeamLocker.Unlock(); // free the thread structure @@ -1921,10 +1936,7 @@ thread_exit(void) panic("thread_exit() called with interrupts disabled!\n"); // boost our priority to get this over with - { - InterruptsSpinLocker _(gSchedulerLock); - scheduler_set_thread_priority(thread, B_URGENT_DISPLAY_PRIORITY); - } + scheduler_set_thread_priority(thread, B_URGENT_DISPLAY_PRIORITY); if (team != kernelTeam) { // Delete all user timers associated with the thread. @@ -1989,14 +2001,17 @@ thread_exit(void) vm_swap_address_space(team->address_space, VMAddressSpace::Kernel()); WriteSpinLocker teamLocker(thread->team_lock); - SpinLocker schedulerLocker(gSchedulerLock); + SpinLocker threadCreationLocker(gThreadCreationLock); // removing the thread and putting its death entry to the parent // team needs to be an atomic operation // remember how long this thread lasted bigtime_t now = system_time(); - InterruptsSpinLocker teamTimeLocker(team->time_lock); + + InterruptsSpinLocker signalLocker(kernelTeam->signal_lock); + SpinLocker teamTimeLocker(kernelTeam->time_lock); SpinLocker threadTimeLocker(thread->time_lock); + thread->kernel_time += now - thread->last_time; thread->last_time = now; @@ -2014,17 +2029,19 @@ thread_exit(void) thread->DeactivateCPUTimeUserTimers(); threadTimeLocker.Unlock(); - teamTimeLocker.Unlock(); // put the thread into the kernel team until it dies remove_thread_from_team(team, thread); insert_thread_into_team(kernelTeam, thread); + teamTimeLocker.Unlock(); + signalLocker.Unlock(); + teamLocker.Unlock(); if (team->death_entry != NULL) { if (--team->death_entry->remaining_threads == 0) - team->death_entry->condition.NotifyOne(true, B_OK); + team->death_entry->condition.NotifyOne(); } if (deleteTeam) { @@ -2032,8 +2049,7 @@ thread_exit(void) // Set the team job control state to "dead" and detach the job // control entry from our team struct. - team_set_job_control_state(team, JOB_CONTROL_STATE_DEAD, NULL, - true); + team_set_job_control_state(team, JOB_CONTROL_STATE_DEAD, NULL); death = team->job_control_entry; team->job_control_entry = NULL; @@ -2050,7 +2066,7 @@ thread_exit(void) death = NULL; } - schedulerLocker.Unlock(); + threadCreationLocker.Unlock(); restore_interrupts(state); threadLocker.Unlock(); @@ -2113,7 +2129,7 @@ thread_exit(void) } } - schedulerLocker.Unlock(); + threadCreationLocker.Unlock(); restore_interrupts(state); threadLocker.Unlock(); @@ -2138,7 +2154,7 @@ thread_exit(void) ThreadLocker threadLocker(thread); state = disable_interrupts(); - SpinLocker schedulerLocker(gSchedulerLock); + SpinLocker threadCreationLocker(gThreadCreationLock); // mark invisible in global hash/list, so it's no longer accessible SpinLocker threadHashLocker(sThreadHashLock); @@ -2156,7 +2172,7 @@ thread_exit(void) select_info* selectInfos = thread->select_infos; thread->select_infos = NULL; - schedulerLocker.Unlock(); + threadCreationLocker.Unlock(); restore_interrupts(state); threadLocker.Unlock(); @@ -2243,7 +2259,7 @@ thread_exit(void) sUndertakerCondition.NotifyOne(); undertakerLocker.Unlock(); - schedulerLocker.Lock(); + SpinLocker schedulerLocker(thread->scheduler_lock); thread->next_state = THREAD_STATE_FREE_ON_RESCHED; scheduler_reschedule(); @@ -2392,7 +2408,7 @@ thread_yield(void) if (thread == NULL) return; - InterruptsSpinLocker _(gSchedulerLock); + InterruptsSpinLocker _(thread->scheduler_lock); thread->has_yielded = true; scheduler_reschedule(); @@ -2604,7 +2620,6 @@ thread_get_io_priority(thread_id id) int32 priority = thread->io_priority; if (priority < 0) { // negative I/O priority means using the (CPU) priority - InterruptsSpinLocker schedulerLocker(gSchedulerLock); priority = thread->priority; } @@ -2798,7 +2813,7 @@ thread_block_timeout(timer* timer) status_t thread_block() { - InterruptsSpinLocker _(gSchedulerLock); + InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); return thread_block_locked(thread_get_current_thread()); } @@ -2835,7 +2850,7 @@ thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout) { Thread* thread = thread_get_current_thread(); - InterruptsSpinLocker locker(gSchedulerLock); + InterruptsSpinLocker locker(thread->scheduler_lock); if (thread->wait.status != 1) return thread->wait.status; @@ -2881,7 +2896,7 @@ thread_block_with_timeout(uint32 timeoutFlags, bigtime_t timeout) void thread_unblock(Thread* thread, status_t status) { - InterruptsSpinLocker _(gSchedulerLock); + InterruptsSpinLocker locker(thread->scheduler_lock); thread_unblock_locked(thread, status); } @@ -2902,7 +2917,7 @@ user_unblock_thread(thread_id threadID, status_t status) if (thread->user_thread == NULL) return B_NOT_ALLOWED; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->scheduler_lock); if (thread->user_thread->wait_status > 0) { thread->user_thread->wait_status = status; @@ -3005,7 +3020,7 @@ _get_thread_info(thread_id id, thread_info *info, size_t size) ThreadLocker threadLocker(thread, true); // fill the info -- also requires the scheduler lock to be held - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->scheduler_lock); fill_thread_info(thread, info, size); @@ -3055,7 +3070,7 @@ _get_next_thread_info(team_id teamID, int32 *_cookie, thread_info *info, *_cookie = lastID; ThreadLocker threadLocker(thread); - InterruptsSpinLocker schedulerLocker(gSchedulerLock); + InterruptsSpinLocker locker(thread->scheduler_lock); fill_thread_info(thread, info, size); @@ -3121,8 +3136,6 @@ rename_thread(thread_id id, const char* name) status_t set_thread_priority(thread_id id, int32 priority) { - int32 oldPriority; - // make sure the passed in priority is within bounds if (priority > THREAD_MAX_SET_PRIORITY) priority = THREAD_MAX_SET_PRIORITY; @@ -3140,12 +3153,7 @@ set_thread_priority(thread_id id, int32 priority) if (thread_is_idle_thread(thread)) return B_NOT_ALLOWED; - InterruptsSpinLocker schedulerLocker(gSchedulerLock); - - oldPriority = thread->priority; - scheduler_set_thread_priority(thread, priority); - - return oldPriority; + return scheduler_set_thread_priority(thread, priority); } From 7e1c4534df199347d63bf7235ced8cc60170bfa7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 8 Nov 2013 03:37:30 +0100 Subject: [PATCH 109/273] libroot: Add adaptive mutex implementation --- headers/private/shared/locks.h | 1 + src/system/libroot/libroot_init.c | 5 +++ src/system/libroot/os/locks/mutex.cpp | 32 ++++++++++++++----- .../libroot/posix/malloc/arch-specific.cpp | 2 +- src/system/runtime_loader/runtime_loader.cpp | 2 ++ 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/headers/private/shared/locks.h b/headers/private/shared/locks.h index 23b227d2c8..27da7fc212 100644 --- a/headers/private/shared/locks.h +++ b/headers/private/shared/locks.h @@ -18,6 +18,7 @@ typedef struct mutex { } mutex; #define MUTEX_FLAG_CLONE_NAME 0x1 +#define MUTEX_FLAG_ADAPTIVE 0x2 #define MUTEX_INITIALIZER(name) { name, 0, 0 } void mutex_init(mutex *lock, const char *name); diff --git a/src/system/libroot/libroot_init.c b/src/system/libroot/libroot_init.c index 867cbd25a6..caef95b2fc 100644 --- a/src/system/libroot/libroot_init.c +++ b/src/system/libroot/libroot_init.c @@ -32,6 +32,7 @@ int __libc_argc; char **__libc_argv; int __gABIVersion; +int32 __gCPUCount; char _single_threaded = true; // determines if I/O locking needed; needed for BeOS compatibility @@ -48,6 +49,7 @@ int _data_offset_main_; void initialize_before(image_id imageID) { + system_info info; char *programPath = __gRuntimeLoader->program_args->args[0]; __gCommPageAddress = __gRuntimeLoader->commpage_address; __gABIVersion = __gRuntimeLoader->abi_version; @@ -70,6 +72,9 @@ initialize_before(image_id imageID) pthread_self()->id = find_thread(NULL); + get_system_info(&info); + __gCPUCount = info.cpu_count; + __init_time((addr_t)__gCommPageAddress); __init_heap(); __init_env(__gRuntimeLoader->program_args); diff --git a/src/system/libroot/os/locks/mutex.cpp b/src/system/libroot/os/locks/mutex.cpp index 2e755bb9c2..7c017e73ff 100644 --- a/src/system/libroot/os/locks/mutex.cpp +++ b/src/system/libroot/os/locks/mutex.cpp @@ -18,6 +18,12 @@ #include +#define MAX_UNSUCCESSFUL_SPINS 100 + + +extern int32 __gCPUCount; + + // #pragma mark - mutex @@ -36,6 +42,9 @@ mutex_init_etc(mutex *lock, const char *name, uint32 flags) lock->name = (flags & MUTEX_FLAG_CLONE_NAME) != 0 ? strdup(name) : name; lock->lock = 0; lock->flags = flags; + + if (__gCPUCount < 2) + lock->flags &= ~uint32(MUTEX_FLAG_ADAPTIVE); } @@ -50,15 +59,22 @@ mutex_destroy(mutex *lock) status_t mutex_lock(mutex *lock) { - // set the locked flag - int32 oldValue = atomic_or(&lock->lock, B_USER_MUTEX_LOCKED); + uint32 count; + const uint32 kMaxCount + = (lock->flags & MUTEX_FLAG_ADAPTIVE) != 0 ? MAX_UNSUCCESSFUL_SPINS : 1; - if ((oldValue & (B_USER_MUTEX_LOCKED | B_USER_MUTEX_WAITING)) == 0 - || (oldValue & B_USER_MUTEX_DISABLED) != 0) { - // No one has the lock or is waiting for it, or the mutex has been - // disabled. - return B_OK; - } + int32 oldValue; + do { + // set the locked flag + oldValue = atomic_or(&lock->lock, B_USER_MUTEX_LOCKED); + + if ((oldValue & (B_USER_MUTEX_LOCKED | B_USER_MUTEX_WAITING)) == 0 + || (oldValue & B_USER_MUTEX_DISABLED) != 0) { + // No one has the lock or is waiting for it, or the mutex has been + // disabled. + return B_OK; + } + } while (count++ < kMaxCount && (oldValue & B_USER_MUTEX_WAITING) != 0); // we have to call the kernel status_t error; diff --git a/src/system/libroot/posix/malloc/arch-specific.cpp b/src/system/libroot/posix/malloc/arch-specific.cpp index 81a39945a8..4b9572e868 100644 --- a/src/system/libroot/posix/malloc/arch-specific.cpp +++ b/src/system/libroot/posix/malloc/arch-specific.cpp @@ -362,7 +362,7 @@ hoardUnsbrk(void *ptr, long size) void hoardLockInit(hoardLockType &lock, const char *name) { - mutex_init(&lock, name); + mutex_init_etc(&lock, name, MUTEX_FLAG_ADAPTIVE); } diff --git a/src/system/runtime_loader/runtime_loader.cpp b/src/system/runtime_loader/runtime_loader.cpp index 82a58ec7dd..14f5eb9936 100644 --- a/src/system/runtime_loader/runtime_loader.cpp +++ b/src/system/runtime_loader/runtime_loader.cpp @@ -24,6 +24,8 @@ struct user_space_program_args *gProgramArgs; void *__gCommPageAddress; +int32 __gCPUCount = 1; + static const char * search_path_for_type(image_type type) From a1feba678d5e14e240183ba775b799384f0708e1 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 11 Nov 2013 21:04:38 +0100 Subject: [PATCH 110/273] kernel/undertaker: Make sure the thread isn't running anymore --- src/system/kernel/thread.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 7b1cdcbcde..a2cebdbb37 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -1112,6 +1112,11 @@ undertaker(void* /*args*/) // we've got an entry Thread* thread = entry.thread; + // make sure the thread isn't running anymore + InterruptsSpinLocker schedulerLocker(thread->scheduler_lock); + ASSERT(thread->state == THREAD_STATE_FREE_ON_RESCHED); + schedulerLocker.Unlock(); + // remove this thread from from the kernel team -- this makes it // unaccessible Team* kernelTeam = team_get_kernel_team(); @@ -2254,12 +2259,13 @@ thread_exit(void) disable_interrupts(); + SpinLocker schedulerLocker(thread->scheduler_lock); + SpinLocker undertakerLocker(sUndertakerLock); sUndertakerEntries.Add(&undertakerEntry); sUndertakerCondition.NotifyOne(); undertakerLocker.Unlock(); - SpinLocker schedulerLocker(thread->scheduler_lock); thread->next_state = THREAD_STATE_FREE_ON_RESCHED; scheduler_reschedule(); From d17b71d6b02775bb918b340c5bbaaa39320dfc53 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 11 Nov 2013 21:46:18 +0100 Subject: [PATCH 111/273] scheduler: Reduce false sharing of per-CPU and per-core data --- src/system/kernel/scheduler/scheduler.cpp | 25 +++++++++++++---------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 6f7a2bf050..84bcbfa2eb 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -45,6 +45,9 @@ #endif +#define CACHE_LINE_ALIGN __attribute__((aligned(64))) + + SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; @@ -87,8 +90,9 @@ struct CPUEntry : public MinMaxHeapLinkImpl { bigtime_t fMeasureTime; int fLoad; -}; -typedef MinMaxHeap CPUHeap; +} CACHE_LINE_ALIGN; +typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; + static CPUEntry* sCPUEntries; static CPUHeap* sCPUPriorityHeaps; @@ -106,17 +110,16 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { bigtime_t fActiveTime; int fLoad; -}; - -static CoreEntry* sCoreEntries; +} CACHE_LINE_ALIGN; typedef Heap, HeapMemberGetLink > CorePriorityHeap; -static CorePriorityHeap* sCorePriorityHeap; - typedef MinMaxHeap, MinMaxHeapMemberGetLink > CoreLoadHeap; + +static CoreEntry* sCoreEntries; +static CorePriorityHeap* sCorePriorityHeap; static CoreLoadHeap* sCoreLoadHeap; static CoreLoadHeap* sCoreHighLoadHeap; @@ -137,7 +140,7 @@ struct PackageEntry : public MinMaxHeapLinkImpl, int32 fIdleCoreCount; int32 fCoreCount; -}; +} CACHE_LINE_ALIGN; typedef MinMaxHeap PackageHeap; typedef DoublyLinkedList IdlePackageList; @@ -150,8 +153,9 @@ static IdlePackageList* sIdlePackageList; // logical processor has its sPinnedRunQueues used for scheduling // pinned threads. typedef RunQueue ThreadRunQueue; -static ThreadRunQueue* sRunQueues; -static ThreadRunQueue* sPinnedRunQueues; + +static ThreadRunQueue* sRunQueues CACHE_LINE_ALIGN; +static ThreadRunQueue* sPinnedRunQueues CACHE_LINE_ALIGN; static int32 sRunQueueCount; // Since CPU IDs used internally by the kernel bear no relation to the actual @@ -160,7 +164,6 @@ static int32 sRunQueueCount; static int32* sCPUToCore; static int32* sCPUToPackage; - struct scheduler_thread_data { scheduler_thread_data() { Init(); } inline void Init(); From e1c40769d32886085323e58a9cc270bc09383e32 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 12 Nov 2013 04:23:42 +0100 Subject: [PATCH 112/273] scheduler: Atomically access time and load measurements --- src/system/kernel/scheduler/scheduler.cpp | 66 +++++++++++------------ 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 84bcbfa2eb..9b41332d54 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -486,7 +486,7 @@ has_cache_expired(Thread* thread) CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; switch (sSchedulerMode) { case SCHEDULER_MODE_LOW_LATENCY: - return coreEntry->fActiveTime + return atomic_get64(&coreEntry->fActiveTime) - schedulerThreadData->went_sleep_active > kCacheExpire; case SCHEDULER_MODE_POWER_SAVING: @@ -945,8 +945,8 @@ compute_cpu_load(int32 cpu) if (oldLoad != sCPUEntries[cpu].fLoad) { int32 core = sCPUToCore[cpu]; - sCoreEntries[core].fLoad -= oldLoad; - sCoreEntries[core].fLoad += sCPUEntries[cpu].fLoad; + int delta = sCPUEntries[cpu].fLoad - oldLoad; + atomic_add(&sCoreEntries[core].fLoad, delta); update_load_heaps(core); } @@ -974,7 +974,8 @@ thread_goes_away(Thread* thread) int32 core = schedulerThreadData->previous_core; schedulerThreadData->went_sleep = system_time(); - schedulerThreadData->went_sleep_active = sCoreEntries[core].fActiveTime; + schedulerThreadData->went_sleep_active + = atomic_get64(&sCoreEntries[core].fActiveTime); } @@ -990,32 +991,32 @@ should_cancel_penalty(Thread* thread) bigtime_t now = system_time(); bigtime_t wentSleep = schedulerThreadData->went_sleep; - if (wentSleep < sCoreEntries[core].fReachedIdle) + if (wentSleep < atomic_get64(&sCoreEntries[core].fReachedIdle)) return true; - if (sCoreEntries[core].fStartedIdle != 0) { - if (wentSleep < sCoreEntries[core].fStartedIdle - && now - sCoreEntries[core].fStartedIdle >= kMinimalWaitTime) { + bigtime_t startedIdle = atomic_get64(&sCoreEntries[core].fStartedIdle); + if (startedIdle != 0) { + if (wentSleep < startedIdle && now - startedIdle >= kMinimalWaitTime) return true; - } - if (wentSleep - sCoreEntries[core].fStartedIdle >= kMinimalWaitTime) + if (wentSleep - startedIdle >= kMinimalWaitTime) return true; } if (get_effective_priority(thread) == B_LOWEST_ACTIVE_PRIORITY) return false; - if (wentSleep < sCoreEntries[core].fReachedIdle) + if (wentSleep < atomic_get64(&sCoreEntries[core].fReachedBottom)) return true; + bigtime_t startedBottom = atomic_get64(&sCoreEntries[core].fStartedBottom); if (sCoreEntries[core].fStartedBottom != 0) { - if (wentSleep < sCoreEntries[core].fStartedBottom - && now - sCoreEntries[core].fStartedBottom >= kMinimalWaitTime) { + if (wentSleep < startedBottom + && now - startedBottom >= kMinimalWaitTime) { return true; } - if (wentSleep - sCoreEntries[core].fStartedBottom >= kMinimalWaitTime) + if (wentSleep - startedBottom >= kMinimalWaitTime) return true; } @@ -1038,17 +1039,8 @@ enqueue(Thread* thread, bool newOne) if (newOne && core >= 0) { int32 priority = get_effective_priority(thread); - if (priority == B_LOWEST_ACTIVE_PRIORITY) { - if (schedulerThreadData->went_sleep - < sCoreEntries[core].fReachedIdle) { - cancel_penalty(thread); - } - } else { - if (schedulerThreadData->went_sleep - < sCoreEntries[core].fReachedBottom) { - cancel_penalty(thread); - } - } + if (should_cancel_penalty(thread)) + cancel_penalty(thread); } int32 threadPriority = get_effective_priority(thread); @@ -1335,13 +1327,16 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) bigtime_t usedTime = now - oldThread->scheduler_data->quantum_start; if (thread_is_idle_thread(oldThread) && usedTime >= kMinimalWaitTime) { - sCoreEntries[thisCore].fReachedBottom = now - kMinimalWaitTime; - sCoreEntries[thisCore].fReachedIdle = now - kMinimalWaitTime; + atomic_set64(&sCoreEntries[thisCore].fReachedBottom, + now - kMinimalWaitTime); + atomic_set64(&sCoreEntries[thisCore].fReachedIdle, + now - kMinimalWaitTime); } if (get_effective_priority(oldThread) == B_LOWEST_ACTIVE_PRIORITY && usedTime >= kMinimalWaitTime) { - sCoreEntries[thisCore].fReachedBottom = now - kMinimalWaitTime; + atomic_set64(&sCoreEntries[thisCore].fReachedBottom, + now - kMinimalWaitTime); } if (!thread_is_idle_thread(oldThread)) { @@ -1351,8 +1346,9 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) atomic_add64(&oldThread->cpu->active_time, active); oldThread->scheduler_data->measure_active_time += active; + sCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - sCoreEntries[thisCore].fActiveTime += active; + atomic_add64(&sCoreEntries[thisCore].fActiveTime, active); } if (!sSingleCore) @@ -1363,16 +1359,16 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) if (thread_is_idle_thread(nextThread)) { if (!thread_is_idle_thread(oldThread)) - sCoreEntries[thisCore].fStartedIdle = now; + atomic_set64(&sCoreEntries[thisCore].fStartedIdle, now); if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - sCoreEntries[thisCore].fStartedBottom = now; + atomic_set64(&sCoreEntries[thisCore].fStartedBottom, now); } else if (nextPriority == B_LOWEST_ACTIVE_PRIORITY) { - sCoreEntries[thisCore].fStartedIdle = 0; + atomic_set64(&sCoreEntries[thisCore].fStartedIdle, 0); if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - sCoreEntries[thisCore].fStartedBottom = now; + atomic_set64(&sCoreEntries[thisCore].fStartedBottom, now); } else { - sCoreEntries[thisCore].fStartedBottom = 0; - sCoreEntries[thisCore].fStartedIdle = 0; + atomic_set64(&sCoreEntries[thisCore].fStartedBottom, 0); + atomic_set64(&sCoreEntries[thisCore].fStartedIdle, 0); } if (!thread_is_idle_thread(nextThread)) { From 8818c942dd31f098825c6230b90515b688d110cf Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 12 Nov 2013 04:26:32 +0100 Subject: [PATCH 113/273] scheduler: Add {CPU,Core,Package}Entry constructors --- src/system/kernel/scheduler/scheduler.cpp | 37 +++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 9b41332d54..1971bb738b 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -84,6 +84,8 @@ static bool (*sShouldRebalance)(Thread* thread); // the core the only real concern is to make lower priority threads give way to // the higher priority threads. struct CPUEntry : public MinMaxHeapLinkImpl { + CPUEntry(); + int32 fCPUNumber; bigtime_t fMeasureActiveTime; @@ -97,6 +99,8 @@ static CPUEntry* sCPUEntries; static CPUHeap* sCPUPriorityHeaps; struct CoreEntry : public DoublyLinkedListLinkImpl { + CoreEntry(); + HeapLink fPriorityHeapLink; MinMaxHeapLink fLoadHeapLink; @@ -134,6 +138,8 @@ static CoreLoadHeap* sCoreHighLoadHeap; // idle are stored in sPackageIdleList (in LIFO manner). struct PackageEntry : public MinMaxHeapLinkImpl, DoublyLinkedListLinkImpl { + PackageEntry(); + int32 fPackageID; DoublyLinkedList fIdleCores; @@ -189,6 +195,31 @@ struct scheduler_thread_data { }; +CPUEntry::CPUEntry() + : + fMeasureActiveTime(0), + fMeasureTime(0), + fLoad(0) +{ +} + + +CoreEntry::CoreEntry() + : + fActiveTime(0), + fLoad(0) +{ +} + + +PackageEntry::PackageEntry() + : + fIdleCoreCount(0), + fCoreCount(0) +{ +} + + void scheduler_thread_data::Init() { @@ -1746,8 +1777,6 @@ _scheduler_init() for (int32 i = 0; i < coreCount; i++) { sCoreEntries[i].fCoreID = i; - sCoreEntries[i].fActiveTime = 0; - sCoreEntries[i].fLoad = 0; status_t result = sCoreLoadHeap->Insert(&sCoreEntries[i], 0); if (result != B_OK) @@ -1767,10 +1796,6 @@ _scheduler_init() for (int32 i = 0; i < cpuCount; i++) { sCPUEntries[i].fCPUNumber = i; - sCPUEntries[i].fMeasureActiveTime = 0; - sCPUEntries[i].fMeasureTime = 0; - sCPUEntries[i].fLoad = 0; - int32 core = sCPUToCore[i]; int32 package = sCPUToPackage[i]; From 829f83632457377650eaf92309fbc856a2276e34 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 12 Nov 2013 04:42:12 +0100 Subject: [PATCH 114/273] scheduler: Minor cleanup --- headers/private/kernel/cpu.h | 2 +- src/system/kernel/scheduler/scheduler.cpp | 109 ++++++++++------------ 2 files changed, 50 insertions(+), 61 deletions(-) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index bcd06aa59b..c1524b1d16 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -55,7 +55,7 @@ typedef struct cpu_ent { int cpu_num; // thread.c: used to force a reschedule at quantum expiration time - int preempted; + bool preempted; timer quantum_timer; // keeping track of CPU activity diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 1971bb738b..9a628c8678 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -834,6 +834,23 @@ choose_cpu(int32 core) } +static void +choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) +{ + if (targetCore == -1 && targetCPU != -1) + targetCore = sCPUToCore[targetCPU]; + else if (targetCore != -1 && targetCPU == -1) + targetCPU = choose_cpu(targetCore); + else if (targetCore == -1 && targetCPU == -1) { + targetCore = choose_core(thread); + targetCPU = choose_cpu(targetCore); + } + + ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); + ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); +} + + static bool should_rebalance_low_latency(Thread* thread) { @@ -1066,53 +1083,32 @@ enqueue(Thread* thread, bool newOne) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - int32 core = schedulerThreadData->previous_core; - if (newOne && core >= 0) { - int32 priority = get_effective_priority(thread); - - if (should_cancel_penalty(thread)) - cancel_penalty(thread); - } - int32 threadPriority = get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); bool pinned = thread->pinned_to_cpu > 0; int32 targetCPU = -1; - int32 targetCore; - if (pinned) { + int32 targetCore = -1; + if (pinned) targetCPU = thread->previous_cpu->cpu_num; - targetCore = sCPUToCore[targetCPU]; - ASSERT(targetCore == schedulerThreadData->previous_core); - } else if (sSingleCore) { + else if (sSingleCore) targetCore = 0; - targetCPU = choose_cpu(targetCore); - - schedulerThreadData->previous_core = targetCore; - } else if (schedulerThreadData->previous_core < 0 + else if (schedulerThreadData->previous_core < 0 || (newOne && has_cache_expired(thread)) || should_rebalance(thread)) { - if (thread_is_idle_thread(thread)) { + if (thread_is_idle_thread(thread)) targetCPU = thread->previous_cpu->cpu_num; - targetCore = sCPUToCore[targetCPU]; - } else { - targetCore = choose_core(thread); - targetCPU = choose_cpu(targetCore); - } - - schedulerThreadData->previous_core = targetCore; - } else { + + } else targetCore = schedulerThreadData->previous_core; - targetCPU = choose_cpu(targetCore); - } - ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); - ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); + choose_core_and_cpu(thread, targetCore, targetCPU); + schedulerThreadData->previous_core = targetCore; - TRACE("enqueueing thread %ld with priority %ld %ld\n", thread->id, - threadPriority, targetCore); + TRACE("enqueueing thread %ld with priority %ld\n", thread->id, + threadPriority); if (pinned) sPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); else @@ -1129,8 +1125,8 @@ enqueue(Thread* thread, bool newOne) Thread* targetThread = gCPU[targetCPU].running_thread; int32 targetPriority = get_effective_priority(targetThread); - TRACE("choosing CPU %ld with current priority %ld\n", targetCPU, - targetPriority); + TRACE("choosing CPU %ld (core %ld) with current priority %ld\n", targetCPU, + targetCore, targetPriority); if (threadPriority > targetPriority) { targetThread->scheduler_data->lost_cpu = true; @@ -1160,6 +1156,17 @@ scheduler_enqueue_in_run_queue(Thread *thread) TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + + int32 core = schedulerThreadData->previous_core; + if (core >= 0) { + int32 priority = get_effective_priority(thread); + + if (should_cancel_penalty(thread)) + cancel_penalty(thread); + } + enqueue(thread, true); } @@ -1169,9 +1176,7 @@ put_back(Thread* thread) { compute_thread_load(thread); - bool pinned = sPinnedRunQueues != NULL && thread->pinned_to_cpu > 0; - - if (pinned) { + if (thread->pinned_to_cpu > 0) { int32 pinnedCPU = thread->previous_cpu->cpu_num; sPinnedRunQueues[pinnedCPU].PushFront(thread, get_effective_priority(thread)); @@ -1240,7 +1245,7 @@ reschedule_event(timer *unused) thread->scheduler_data->lost_cpu = true; thread->cpu->invoke_scheduler = true; - thread->cpu->preempted = 1; + thread->cpu->preempted = true; return B_HANDLED_INTERRUPT; } @@ -1324,14 +1329,11 @@ static inline Thread* dequeue_thread(int32 thisCPU) { int32 thisCore = sCPUToCore[thisCPU]; + Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); + Thread* pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); - Thread* pinnedThread = NULL; - if (sPinnedRunQueues != NULL) - pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); - - if (sharedThread == NULL && pinnedThread == NULL) - return NULL; + ASSERT(sharedThread != NULL || pinnedThread != NULL); int32 pinnedPriority = -1; if (pinnedThread != NULL) @@ -1497,20 +1499,7 @@ _scheduler_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority - if (oldThread->cpu->disabled) { - ASSERT(sPinnedRunQueues != NULL); - nextThread = sPinnedRunQueues[thisCPU].PeekMaximum(); - if (nextThread != NULL) - sPinnedRunQueues[thisCPU].Remove(nextThread); - else { - nextThread = sRunQueues[thisCore].GetHead(B_IDLE_PRIORITY); - if (nextThread != NULL) - sRunQueues[thisCore].Remove(nextThread); - } - } else - nextThread = dequeue_thread(thisCPU); - if (!nextThread) - panic("reschedule(): run queues are empty!\n"); + nextThread = dequeue_thread(thisCPU); if (nextThread != oldThread) acquire_spinlock(&nextThread->scheduler_lock); @@ -1543,7 +1532,7 @@ _scheduler_reschedule(void) if (!oldThread->cpu->preempted) cancel_timer(quantumTimer); - oldThread->cpu->preempted = 0; + oldThread->cpu->preempted = false; if (!thread_is_idle_thread(nextThread)) { bigtime_t quantum = compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, @@ -1621,7 +1610,7 @@ scheduler_set_operation_mode(scheduler_mode mode) return B_BAD_VALUE; } - const char* modeNames[] = { "performance", "power saving" }; + const char* modeNames[] = { "low latency", "power saving" }; dprintf("scheduler: switching to %s mode\n", modeNames[mode]); InterruptsSpinLocker _(sSchedulerInternalLock); From 5f3a65e5780002f8a4acf814a1b3347e95e65bd2 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 13 Nov 2013 00:01:02 +0100 Subject: [PATCH 115/273] scheduler: Remove sCorePriorityHeap sCorePriorityHeap was meant to be a temporary solution anyway. Thread migration and assignment is now entirely based on core load. --- src/system/kernel/scheduler/scheduler.cpp | 168 ++++++++-------------- 1 file changed, 56 insertions(+), 112 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 9a628c8678..735cc52753 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -115,15 +115,11 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { int fLoad; } CACHE_LINE_ALIGN; -typedef Heap, - HeapMemberGetLink > - CorePriorityHeap; typedef MinMaxHeap, MinMaxHeapMemberGetLink > CoreLoadHeap; static CoreEntry* sCoreEntries; -static CorePriorityHeap* sCorePriorityHeap; static CoreLoadHeap* sCoreLoadHeap; static CoreLoadHeap* sCoreHighLoadHeap; @@ -390,33 +386,6 @@ dump_core_load_heap(CoreLoadHeap* heap) static int dump_cpu_heap(int argc, char** argv) { - CorePriorityHeap temp(sRunQueueCount); - - CoreEntry* entry = sCorePriorityHeap->PeekRoot(); - if (entry != NULL) - kprintf("core priority\n"); - else - kprintf("No active cores.\n"); - - while (entry) { - int32 core = entry->fCoreID; - int32 key = CorePriorityHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %8" B_PRId32 "\n", core, key); - - sCorePriorityHeap->RemoveRoot(); - temp.Insert(entry, key); - - entry = sCorePriorityHeap->PeekRoot(); - } - - entry = temp.PeekRoot(); - while (entry) { - int32 key = CorePriorityHeap::GetKey(entry); - temp.RemoveRoot(); - sCorePriorityHeap->Insert(entry, key); - entry = temp.PeekRoot(); - } - kprintf("\ncore load\n"); dump_core_load_heap(sCoreLoadHeap); kprintf("---------\n"); @@ -664,6 +633,8 @@ update_priority_heaps(int32 cpu, int32 priority) { int32 core = sCPUToCore[cpu]; + int32 corePriority = CPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); + sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); if (sSingleCore) @@ -671,72 +642,63 @@ update_priority_heaps(int32 cpu, int32 priority) int32 maxPriority = CPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); - int32 corePriority = CorePriorityHeap::GetKey(&sCoreEntries[core]); - if (corePriority != maxPriority) { - if (maxPriority == B_IDLE_PRIORITY) { - sCorePriorityHeap->ModifyKey(&sCoreEntries[core], B_IDLE_PRIORITY); - ASSERT(sCorePriorityHeap->PeekRoot() == &sCoreEntries[core]); - sCorePriorityHeap->RemoveRoot(); - } else if (corePriority == B_IDLE_PRIORITY) - sCorePriorityHeap->Insert(&sCoreEntries[core], maxPriority); - else - sCorePriorityHeap->ModifyKey(&sCoreEntries[core], maxPriority); + if (corePriority == maxPriority) + return; - int32 package = sCPUToPackage[cpu]; - PackageEntry* packageEntry = &sPackageEntries[package]; - if (maxPriority == B_IDLE_PRIORITY) { - // core goes idle - ASSERT(packageEntry->fIdleCoreCount >= 0); - ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); + int32 package = sCPUToPackage[cpu]; + PackageEntry* packageEntry = &sPackageEntries[package]; + if (maxPriority == B_IDLE_PRIORITY) { + // core goes idle + ASSERT(packageEntry->fIdleCoreCount >= 0); + ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); - packageEntry->fIdleCoreCount++; - packageEntry->fIdleCores.Add(&sCoreEntries[core]); + packageEntry->fIdleCoreCount++; + packageEntry->fIdleCores.Add(&sCoreEntries[core]); - if (packageEntry->fIdleCoreCount == 1) { - // first core on that package to go idle - - if (packageEntry->fCoreCount > 1) - sPackageUsageHeap->Insert(packageEntry, 1); - else - sIdlePackageList->Add(packageEntry); - } else if (packageEntry->fIdleCoreCount - == packageEntry->fCoreCount) { - // package goes idle - sPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); - sPackageUsageHeap->RemoveMinimum(); + if (packageEntry->fIdleCoreCount == 1) { + // first core on that package to go idle + if (packageEntry->fCoreCount > 1) + sPackageUsageHeap->Insert(packageEntry, 1); + else sIdlePackageList->Add(packageEntry); - } else { - sPackageUsageHeap->ModifyKey(packageEntry, - packageEntry->fIdleCoreCount); - } - } else if (corePriority == B_IDLE_PRIORITY) { - // core wakes up - ASSERT(packageEntry->fIdleCoreCount > 0); - ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount--; - packageEntry->fIdleCores.Remove(&sCoreEntries[core]); - - if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { - // package wakes up - sIdlePackageList->Remove(packageEntry); - - if (packageEntry->fIdleCoreCount > 0) { - sPackageUsageHeap->Insert(packageEntry, - packageEntry->fIdleCoreCount); - } - } else if (packageEntry->fIdleCoreCount == 0) { - // no more idle cores in the package - sPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); - sPackageUsageHeap->RemoveMinimum(); - } else { - sPackageUsageHeap->ModifyKey(packageEntry, + } else if (packageEntry->fIdleCoreCount + == packageEntry->fCoreCount) { + // package goes idle + sPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); + sPackageUsageHeap->RemoveMinimum(); + + sIdlePackageList->Add(packageEntry); + } else { + sPackageUsageHeap->ModifyKey(packageEntry, + packageEntry->fIdleCoreCount); + } + } else if (corePriority == B_IDLE_PRIORITY) { + // core wakes up + ASSERT(packageEntry->fIdleCoreCount > 0); + ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount--; + packageEntry->fIdleCores.Remove(&sCoreEntries[core]); + + if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { + // package wakes up + sIdlePackageList->Remove(packageEntry); + + if (packageEntry->fIdleCoreCount > 0) { + sPackageUsageHeap->Insert(packageEntry, packageEntry->fIdleCoreCount); } + } else if (packageEntry->fIdleCoreCount == 0) { + // no more idle cores in the package + sPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); + sPackageUsageHeap->RemoveMinimum(); + } else { + sPackageUsageHeap->ModifyKey(packageEntry, + packageEntry->fIdleCoreCount); } } } @@ -757,14 +719,9 @@ choose_core_low_latency(Thread* thread) entry = package->fIdleCores.Last(); } else { // no idle cores, use least occupied core - entry = sCorePriorityHeap->PeekRoot(); - - int32 priority = get_effective_priority(thread); - if (CorePriorityHeap::GetKey(entry) >= priority) { - entry = sCoreLoadHeap->PeekMinimum(); - if (entry == NULL) - entry = sCoreHighLoadHeap->PeekMinimum(); - } + entry = sCoreLoadHeap->PeekMinimum(); + if (entry == NULL) + entry = sCoreHighLoadHeap->PeekMinimum(); } ASSERT(entry != NULL); @@ -792,11 +749,9 @@ choose_core_power_saving(Thread* thread) if (sSmallTaskCore < 0) sSmallTaskCore = sCoreLoadHeap->PeekMaximum()->fCoreID; entry = &sCoreEntries[sSmallTaskCore]; - } else if (sCorePriorityHeap->PeekRoot() != NULL - && CorePriorityHeap::GetKey(sCorePriorityHeap->PeekRoot()) - < priority) { + } else if (sCoreLoadHeap->PeekMinimum() != NULL) { // run immediately on already woken core - entry = sCorePriorityHeap->PeekRoot(); + entry = sCoreLoadHeap->PeekMinimum(); } else if (sPackageUsageHeap->PeekMinimum() != NULL) { // wake new core PackageEntry* package = sPackageUsageHeap->PeekMinimum(); @@ -1749,11 +1704,6 @@ _scheduler_init() return B_NO_MEMORY; ArrayDeleter coreEntriesDeleter(sCoreEntries); - sCorePriorityHeap = new CorePriorityHeap(coreCount); - if (sCorePriorityHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter corePriorityHeapDeleter(sCorePriorityHeap); - sCoreLoadHeap = new CoreLoadHeap; if (sCoreLoadHeap == NULL) return B_NO_MEMORY; @@ -1770,11 +1720,6 @@ _scheduler_init() status_t result = sCoreLoadHeap->Insert(&sCoreEntries[i], 0); if (result != B_OK) return result; - - result = sCorePriorityHeap->Insert(&sCoreEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; - sCorePriorityHeap->RemoveRoot(); } sCPUPriorityHeaps = new CPUHeap[coreCount]; @@ -1845,7 +1790,6 @@ _scheduler_init() pinnedRunQueuesDeleter.Detach(); coreHighLoadHeapDeleter.Detach(); coreLoadHeapDeleter.Detach(); - corePriorityHeapDeleter.Detach(); cpuPriorityHeapDeleter.Detach(); coreEntriesDeleter.Detach(); cpuEntriesDeleter.Detach(); From 72e1b394a40aa34fd93fc83e9137dfbe48a0e34e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 13 Nov 2013 00:36:48 +0100 Subject: [PATCH 116/273] scheduler: Fix gcc2 build --- src/system/kernel/scheduler/scheduler.cpp | 42 +++++++++-------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 735cc52753..0320363fcc 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -91,19 +91,17 @@ struct CPUEntry : public MinMaxHeapLinkImpl { bigtime_t fMeasureActiveTime; bigtime_t fMeasureTime; - int fLoad; + int32 fLoad; } CACHE_LINE_ALIGN; typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; static CPUEntry* sCPUEntries; static CPUHeap* sCPUPriorityHeaps; -struct CoreEntry : public DoublyLinkedListLinkImpl { +struct CoreEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { CoreEntry(); - HeapLink fPriorityHeapLink; - MinMaxHeapLink fLoadHeapLink; - int32 fCoreID; bigtime_t fStartedBottom; @@ -113,11 +111,9 @@ struct CoreEntry : public DoublyLinkedListLinkImpl { bigtime_t fActiveTime; - int fLoad; + int32 fLoad; } CACHE_LINE_ALIGN; -typedef MinMaxHeap, - MinMaxHeapMemberGetLink > - CoreLoadHeap; +typedef MinMaxHeap CoreLoadHeap; static CoreEntry* sCoreEntries; static CoreLoadHeap* sCoreLoadHeap; @@ -182,7 +178,7 @@ struct scheduler_thread_data { bigtime_t measure_active_time; bigtime_t measure_time; - int load; + int32 load; bigtime_t went_sleep; bigtime_t went_sleep_active; @@ -336,7 +332,7 @@ dump_heap(CPUHeap* heap) while (entry) { int32 cpu = entry->fCPUNumber; int32 key = CPUHeap::GetKey(entry); - kprintf("%3" B_PRId32 " %8" B_PRId32 " %3d%%\n", cpu, key, + kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, sCPUEntries[cpu].fLoad / 10); heap->RemoveMinimum(); @@ -363,7 +359,7 @@ dump_core_load_heap(CoreLoadHeap* heap) CoreEntry* entry = heap->PeekMinimum(); while (entry) { - int key = CoreLoadHeap::GetKey(entry); + int32 key = CoreLoadHeap::GetKey(entry); kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, entry->fLoad / cpuPerCore / 10); @@ -375,7 +371,7 @@ dump_core_load_heap(CoreLoadHeap* heap) entry = temp.PeekMinimum(); while (entry) { - int key = CoreLoadHeap::GetKey(entry); + int32 key = CoreLoadHeap::GetKey(entry); temp.RemoveMinimum(); heap->Insert(entry, key); entry = temp.PeekMinimum(); @@ -517,7 +513,7 @@ scheduler_dump_thread_data(Thread* thread) additionalPenalty, schedulerThreadData->additional_penalty); kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", schedulerThreadData->stolen_time); - kprintf("\tload:\t\t\t%d%%\n", schedulerThreadData->load / 10); + kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", schedulerThreadData->load / 10); kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", schedulerThreadData->went_sleep); kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", @@ -539,8 +535,8 @@ update_load_heaps(int32 core) CoreEntry* entry = &sCoreEntries[core]; int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; - int newKey = entry->fLoad / cpuPerCore; - int oldKey = CoreLoadHeap::GetKey(entry); + int32 newKey = entry->fLoad / cpuPerCore; + int32 oldKey = CoreLoadHeap::GetKey(entry); ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); ASSERT(newKey >= 0 && newKey <= kMaxLoad); @@ -741,8 +737,6 @@ choose_core_power_saving(Thread* thread) { CoreEntry* entry; - int32 priority = get_effective_priority(thread); - if (is_small_task_packing_enabled() && is_task_small(thread) && sCoreLoadHeap->PeekMaximum() != NULL) { // try to pack all threads on one core @@ -892,7 +886,7 @@ should_rebalance(Thread* thread) static inline int -compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int& load) +compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load) { const bigtime_t kLoadMeasureInterval = 50000; const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; @@ -948,7 +942,7 @@ compute_cpu_load(int32 cpu) if (oldLoad != sCPUEntries[cpu].fLoad) { int32 core = sCPUToCore[cpu]; - int delta = sCPUEntries[cpu].fLoad - oldLoad; + int32 delta = sCPUEntries[cpu].fLoad - oldLoad; atomic_add(&sCoreEntries[core].fLoad, delta); update_load_heaps(core); @@ -1116,8 +1110,6 @@ scheduler_enqueue_in_run_queue(Thread *thread) int32 core = schedulerThreadData->previous_core; if (core >= 0) { - int32 priority = get_effective_priority(thread); - if (should_cancel_penalty(thread)) cancel_penalty(thread); } @@ -1369,12 +1361,12 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) static inline void update_cpu_performance(Thread* thread, int32 thisCore) { - int load = max_c(thread->scheduler_data->load, + int32 load = max_c(thread->scheduler_data->load, sCoreEntries[thisCore].fLoad); load = min_c(max_c(load, 0), kMaxLoad); if (load < kTargetLoad) { - int delta = kTargetLoad - load; + int32 delta = kTargetLoad - load; delta *= kTargetLoad; delta /= kCPUPerformanceScaleMax; @@ -1384,7 +1376,7 @@ update_cpu_performance(Thread* thread, int32 thisCore) bool allowBoost = sSchedulerMode != SCHEDULER_MODE_POWER_SAVING; allowBoost = allowBoost || thread->scheduler_data->priority_penalty > 0; - int delta = load - kTargetLoad; + int32 delta = load - kTargetLoad; delta *= kMaxLoad - kTargetLoad; delta /= kCPUPerformanceScaleMax; From 288a2664a2de429f159d746beaab87373184cd3d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 13 Nov 2013 05:31:58 +0100 Subject: [PATCH 117/273] scheduler: Remove sSchedulerInternalLock * pin idle threads to their specific CPUs * allow scheduler to implement SMP_MSG_RESCHEDULE handler * scheduler_set_thread_priority() reworked * at reschedule: enqueue old thread after dequeueing the new one --- headers/private/kernel/kscheduler.h | 4 +- src/system/kernel/scheduler/scheduler.cpp | 256 +++++++++++++++------- src/system/kernel/smp.cpp | 5 +- src/system/kernel/thread.cpp | 3 +- 4 files changed, 185 insertions(+), 83 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index bcb5fd6e0d..addd4583cd 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -34,6 +34,8 @@ extern "C" { */ void scheduler_enqueue_in_run_queue(Thread* thread); +void scheduler_reschedule_ici(void); + /*! Selects a thread from the ready-to-run queue and, if that's not the calling thread, switches the current CPU's context to run the selected thread. @@ -59,7 +61,7 @@ status_t scheduler_on_thread_create(Thread* thread, bool idleThread); use. The per-thread housekeeping data structures are reset, if needed. */ -void scheduler_on_thread_init(Thread* thread); +void scheduler_on_thread_init(Thread* thread); /*! Called when a Thread structure is freed. Frees up any per-thread resources allocated on the scheduler's part. The diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 0320363fcc..d8a3d98f8d 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -51,7 +51,6 @@ SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -static spinlock sSchedulerInternalLock; static bool sSchedulerEnabled; const bigtime_t kThreadQuantum = 1000; @@ -73,6 +72,7 @@ static int32 sSmallTaskCore; static bool sSingleCore; static scheduler_mode sSchedulerMode; +static rw_spinlock sSchedulerModeLock = B_RW_SPINLOCK_INITIALIZER; static int32 (*sChooseCore)(Thread* thread); static bool (*sShouldRebalance)(Thread* thread); @@ -88,6 +88,8 @@ struct CPUEntry : public MinMaxHeapLinkImpl { int32 fCPUNumber; + int32 fPriority; + bigtime_t fMeasureActiveTime; bigtime_t fMeasureTime; @@ -104,6 +106,8 @@ struct CoreEntry : public MinMaxHeapLinkImpl, int32 fCoreID; + spinlock fLock; + bigtime_t fStartedBottom; bigtime_t fReachedBottom; bigtime_t fStartedIdle; @@ -118,6 +122,7 @@ typedef MinMaxHeap CoreLoadHeap; static CoreEntry* sCoreEntries; static CoreLoadHeap* sCoreLoadHeap; static CoreLoadHeap* sCoreHighLoadHeap; +static spinlock sCoreHeapsLock = B_SPINLOCK_INITIALIZER; // sPackageUsageHeap is used to decide which core should be woken up from the // idle state. When aiming for performance we should use as many packages as @@ -145,6 +150,7 @@ typedef DoublyLinkedList IdlePackageList; static PackageEntry* sPackageEntries; static PackageHeap* sPackageUsageHeap; static IdlePackageList* sIdlePackageList; +static spinlock sIdlePackageLock = B_SPINLOCK_INITIALIZER; // The run queues. Holds the threads ready to run ordered by priority. // One queue per schedulable target per core. Additionally, each @@ -184,11 +190,14 @@ struct scheduler_thread_data { bigtime_t went_sleep_active; int32 previous_core; + + bool enqueued; }; CPUEntry::CPUEntry() : + fPriority(B_IDLE_PRIORITY), fMeasureActiveTime(0), fMeasureTime(0), fLoad(0) @@ -201,6 +210,7 @@ CoreEntry::CoreEntry() fActiveTime(0), fLoad(0) { + B_INITIALIZE_SPINLOCK(&fLock); } @@ -232,6 +242,7 @@ scheduler_thread_data::Init() cpu_bound = true; previous_core = -1; + enqueued = false; } @@ -534,6 +545,8 @@ update_load_heaps(int32 core) CoreEntry* entry = &sCoreEntries[core]; + SpinLocker coreLocker(sCoreHeapsLock); + int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; int32 newKey = entry->fLoad / cpuPerCore; int32 oldKey = CoreLoadHeap::GetKey(entry); @@ -625,12 +638,13 @@ cancel_penalty(Thread* thread) static inline void -update_priority_heaps(int32 cpu, int32 priority) +update_cpu_priority(int32 cpu, int32 priority) { int32 core = sCPUToCore[cpu]; int32 corePriority = CPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); + sCPUEntries[cpu].fPriority = priority; sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); if (sSingleCore) @@ -645,6 +659,8 @@ update_priority_heaps(int32 cpu, int32 priority) int32 package = sCPUToPackage[cpu]; PackageEntry* packageEntry = &sPackageEntries[package]; if (maxPriority == B_IDLE_PRIORITY) { + SpinLocker _(sIdlePackageLock); + // core goes idle ASSERT(packageEntry->fIdleCoreCount >= 0); ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); @@ -672,6 +688,8 @@ update_priority_heaps(int32 cpu, int32 priority) packageEntry->fIdleCoreCount); } } else if (corePriority == B_IDLE_PRIORITY) { + SpinLocker _(sIdlePackageLock); + // core wakes up ASSERT(packageEntry->fIdleCoreCount > 0); ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); @@ -783,9 +801,11 @@ choose_cpu(int32 core) } -static void +static bool choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) { + SpinLocker coreLocker(sCoreHeapsLock); + if (targetCore == -1 && targetCPU != -1) targetCore = sCPUToCore[targetCPU]; else if (targetCore != -1 && targetCPU == -1) @@ -797,6 +817,19 @@ choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); + + int32 targetPriority = sCPUEntries[targetCPU].fPriority; + int32 threadPriority = get_effective_priority(thread); + + if (threadPriority > targetPriority) { + // It is possible that another CPU schedules the thread before the + // target CPU. However, since the target CPU is sent an ICI it will + // reschedule anyway and update its heap key to the correct value. + update_cpu_priority(targetCPU, threadPriority); + return true; + } + + return false; } @@ -816,6 +849,8 @@ should_rebalance_low_latency(Thread* thread) // If there is high load on this core but this thread does not contribute // significantly consider giving it to someone less busy. if (coreEntry->fLoad > kHighLoad) { + SpinLocker coreLocker(sCoreHeapsLock); + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference) return true; @@ -823,6 +858,8 @@ should_rebalance_low_latency(Thread* thread) // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... + SpinLocker coreLocker(sCoreHeapsLock); + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); if (other == NULL) other = sCoreHighLoadHeap->PeekMinimum(); @@ -866,6 +903,8 @@ should_rebalance_power_saving(Thread* thread) // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... + SpinLocker coreLocker(sCoreHeapsLock); + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); if (other == NULL) other = sCoreHighLoadHeap->PeekMinimum(); @@ -1031,7 +1070,8 @@ enqueue(Thread* thread, bool newOne) compute_thread_load(thread); scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - + schedulerThreadData->cpu_bound = true; + schedulerThreadData->time_left = 0; int32 threadPriority = get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); @@ -1043,48 +1083,31 @@ enqueue(Thread* thread, bool newOne) targetCPU = thread->previous_cpu->cpu_num; else if (sSingleCore) targetCore = 0; - else if (schedulerThreadData->previous_core < 0 - || (newOne && has_cache_expired(thread)) - || should_rebalance(thread)) { - - if (thread_is_idle_thread(thread)) - targetCPU = thread->previous_cpu->cpu_num; - - } else + else if (schedulerThreadData->previous_core >= 0 + && (!newOne || !has_cache_expired(thread)) + && !should_rebalance(thread)) { targetCore = schedulerThreadData->previous_core; + } - choose_core_and_cpu(thread, targetCore, targetCPU); + bool shouldReschedule = choose_core_and_cpu(thread, targetCore, targetCPU); schedulerThreadData->previous_core = targetCore; - TRACE("enqueueing thread %ld with priority %ld\n", thread->id, - threadPriority); + TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", + thread->id, threadPriority, targetCPU, targetCore); + + SpinLocker runQueueLocker(sCoreEntries[targetCore].fLock); + thread->scheduler_data->enqueued = true; if (pinned) sPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); else sRunQueues[targetCore].PushBack(thread, threadPriority); - - schedulerThreadData->cpu_bound = true; - schedulerThreadData->time_left = 0; - schedulerThreadData->stolen_time = 0; + runQueueLocker.Unlock(); // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - Thread* targetThread = gCPU[targetCPU].running_thread; - int32 targetPriority = get_effective_priority(targetThread); - - TRACE("choosing CPU %ld (core %ld) with current priority %ld\n", targetCPU, - targetCore, targetPriority); - - if (threadPriority > targetPriority) { - targetThread->scheduler_data->lost_cpu = true; - - // It is possible that another CPU schedules the thread before the - // target CPU. However, since the target CPU is sent an ICI it will - // reschedule anyway and update its heap key to the correct value. - update_priority_heaps(targetCPU, threadPriority); - + if (shouldReschedule) { if (targetCPU == smp_get_current_cpu()) gCPU[targetCPU].invoke_scheduler = true; else { @@ -1101,7 +1124,7 @@ enqueue(Thread* thread, bool newOne) void scheduler_enqueue_in_run_queue(Thread *thread) { - InterruptsSpinLocker _(sSchedulerInternalLock); + InterruptsReadSpinLocker modeLocker(sSchedulerModeLock); TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); @@ -1123,13 +1146,21 @@ put_back(Thread* thread) { compute_thread_load(thread); + int32 core = sCPUToCore[smp_get_current_cpu()]; + + SpinLocker runQueueLocker(sCoreEntries[core].fLock); + thread->scheduler_data->enqueued = true; if (thread->pinned_to_cpu > 0) { int32 pinnedCPU = thread->previous_cpu->cpu_num; + + ASSERT(pinnedCPU == smp_get_current_cpu()); sPinnedRunQueues[pinnedCPU].PushFront(thread, get_effective_priority(thread)); } else { int32 previousCore = thread->scheduler_data->previous_core; ASSERT(previousCore >= 0); + + ASSERT(previousCore == core); sRunQueues[previousCore].PushFront(thread, get_effective_priority(thread)); } @@ -1141,58 +1172,88 @@ put_back(Thread* thread) int32 scheduler_set_thread_priority(Thread *thread, int32 priority) { - InterruptsSpinLocker _(sSchedulerInternalLock); - - if (priority == thread->priority) - return thread->priority; + InterruptsSpinLocker _(thread->scheduler_lock); + InterruptsReadSpinLocker modeLocker(sSchedulerModeLock); int32 oldPriority = thread->priority; TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", thread->id, priority, oldPriority, get_effective_priority(thread)); + cancel_penalty(thread); + + if (priority == thread->priority) + return thread->priority; + + thread->priority = priority; + if (thread->state != B_THREAD_READY) { cancel_penalty(thread); thread->priority = priority; - if (thread->state == B_THREAD_RUNNING) - update_priority_heaps(thread->cpu->cpu_num, priority); + if (thread->state == B_THREAD_RUNNING) { + SpinLocker coreLocker(sCoreHeapsLock); + update_cpu_priority(thread->cpu->cpu_num, priority); + } return oldPriority; } // The thread is in the run queue. We need to remove it and re-insert it at // a new position. - T(RemoveThread(thread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, - thread); - - // remove thread from run queue + bool pinned = thread->pinned_to_cpu > 0; + int32 previousCPU = thread->previous_cpu->cpu_num; int32 previousCore = thread->scheduler_data->previous_core; ASSERT(previousCore >= 0); - sRunQueues[previousCore].Remove(thread); - // set priority and re-insert - cancel_penalty(thread); - thread->priority = priority; - enqueue(thread, true); + SpinLocker runQueueLocker(sCoreEntries[previousCore].fLock); + + // the thread might have been already dequeued and is about to start + // running once we release its scheduler_lock, in such case we can not + // attempt to dequeue it + if (thread->scheduler_data->enqueued) { + T(RemoveThread(thread)); + + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, + thread); + + thread->scheduler_data->enqueued = false; + if (pinned) + sPinnedRunQueues[previousCPU].Remove(thread); + else + sRunQueues[previousCore].Remove(thread); + runQueueLocker.Unlock(); + + enqueue(thread, true); + } return oldPriority; } -static int32 -reschedule_event(timer *unused) +static inline void +reschedule_needed() { - // This function is called as a result of the timer event set by the - // scheduler. Make sure the reschedule() is invoked. - Thread* thread= thread_get_current_thread(); + // This function is called as a result of either the timer event set by the + // scheduler or an incoming ICI. Make sure the reschedule() is invoked. + thread_get_current_thread()->scheduler_data->lost_cpu = true; + get_cpu_struct()->invoke_scheduler = true; +} - thread->scheduler_data->lost_cpu = true; - thread->cpu->invoke_scheduler = true; - thread->cpu->preempted = true; + +void +scheduler_reschedule_ici() +{ + reschedule_needed(); +} + + +static int32 +reschedule_event(timer* /* unused */) +{ + reschedule_needed(); + get_cpu_struct()->preempted = true; return B_HANDLED_INTERRUPT; } @@ -1273,14 +1334,16 @@ compute_quantum(Thread* thread) static inline Thread* -dequeue_thread(int32 thisCPU) +choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) { int32 thisCore = sCPUToCore[thisCPU]; + SpinLocker runQueueLocker(sCoreEntries[thisCore].fLock); + Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); Thread* pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); - ASSERT(sharedThread != NULL || pinnedThread != NULL); + ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); int32 pinnedPriority = -1; if (pinnedThread != NULL) @@ -1290,11 +1353,27 @@ dequeue_thread(int32 thisCPU) if (sharedThread != NULL) sharedPriority = get_effective_priority(sharedThread); + int32 oldPriority = -1; + if (oldThread != NULL) + oldPriority = get_effective_priority(oldThread); + + int32 rest = max_c(pinnedPriority, sharedPriority); + if (oldPriority > rest || (!putAtBack && oldPriority == rest)) { + ASSERT(!oldThread->scheduler_data->enqueued); + return oldThread; + } + if (sharedPriority > pinnedPriority) { + ASSERT(sharedThread->scheduler_data->enqueued); + sharedThread->scheduler_data->enqueued = false; + sRunQueues[thisCore].Remove(sharedThread); return sharedThread; } + ASSERT(pinnedThread->scheduler_data->enqueued); + pinnedThread->scheduler_data->enqueued = false; + sPinnedRunQueues[thisCPU].Remove(pinnedThread); return pinnedThread; } @@ -1388,7 +1467,7 @@ update_cpu_performance(Thread* thread, int32 thisCore) static void _scheduler_reschedule(void) { - InterruptsSpinLocker internalLocker(sSchedulerInternalLock); + InterruptsReadSpinLocker modeLocker(sSchedulerModeLock); Thread* oldThread = thread_get_current_thread(); @@ -1401,14 +1480,13 @@ _scheduler_reschedule(void) oldThread->state = oldThread->next_state; scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; - // update CPU heap so that old thread would have CPU properly chosen - Thread* nextThread = sRunQueues[thisCore].PeekMaximum(); - if (nextThread != NULL) - update_priority_heaps(thisCPU, get_effective_priority(nextThread)); - + bool enqueueOldThread = false; + bool putOldThreadAtBack = false; switch (oldThread->next_state) { case B_THREAD_RUNNING: case B_THREAD_READY: + enqueueOldThread = true; + if (!schedulerOldThreadData->lost_cpu) schedulerOldThreadData->cpu_bound = false; @@ -1419,11 +1497,11 @@ _scheduler_reschedule(void) TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, get_effective_priority(oldThread)); - enqueue(oldThread, false); + putOldThreadAtBack = true; } else { TRACE("putting thread %ld back in run queue priority = %ld\n", oldThread->id, get_effective_priority(oldThread)); - put_back(oldThread); + putOldThreadAtBack = false; } break; @@ -1445,10 +1523,20 @@ _scheduler_reschedule(void) oldThread->has_yielded = false; schedulerOldThreadData->lost_cpu = false; - // select thread with the biggest priority - nextThread = dequeue_thread(thisCPU); - if (nextThread != oldThread) + // select thread with the biggest priority and enqueue back the old thread + Thread* nextThread + = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, + putOldThreadAtBack); + if (nextThread != oldThread) { + if (enqueueOldThread) { + if (putOldThreadAtBack) + enqueue(oldThread, false); + else + put_back(oldThread); + } + acquire_spinlock(&nextThread->scheduler_lock); + } TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, nextThread->id); @@ -1460,12 +1548,16 @@ _scheduler_reschedule(void) oldThread, nextThread); // update CPU heap - update_priority_heaps(thisCPU, - get_effective_priority(nextThread)); + { + SpinLocker coreLocker(sCoreHeapsLock); + update_cpu_priority(thisCPU, get_effective_priority(nextThread)); + } nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; + ASSERT(nextThread->scheduler_data->previous_core == thisCore); + compute_thread_load(nextThread); // track kernel time (user time is tracked in thread_at_kernel_entry()) @@ -1489,7 +1581,7 @@ _scheduler_reschedule(void) } else nextThread->scheduler_data->quantum_start = system_time(); - internalLocker.Unlock(); + modeLocker.Unlock(); if (nextThread != oldThread) scheduler_switch_thread(oldThread, nextThread); } @@ -1527,6 +1619,16 @@ void scheduler_on_thread_init(Thread* thread) { thread->scheduler_data->Init(); + + if (thread_is_idle_thread(thread)) { + static int32 sIdleThreadsID; + int32 cpu = atomic_add(&sIdleThreadsID, 1); + + thread->previous_cpu = &gCPU[cpu]; + thread->pinned_to_cpu = 1; + + thread->scheduler_data->previous_core = sCPUToCore[cpu]; + } } @@ -1560,7 +1662,7 @@ scheduler_set_operation_mode(scheduler_mode mode) const char* modeNames[] = { "low latency", "power saving" }; dprintf("scheduler: switching to %s mode\n", modeNames[mode]); - InterruptsSpinLocker _(sSchedulerInternalLock); + InterruptsWriteSpinLocker _(sSchedulerModeLock); sSchedulerMode = mode; switch (mode) { diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 950b499a9b..d73f787a8f 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -874,11 +874,8 @@ process_pending_ici(int32 currentCPU) break; } case SMP_MSG_RESCHEDULE: - { - cpu_ent* cpu = thread_get_current_thread()->cpu; - cpu->invoke_scheduler = true; + scheduler_reschedule_ici(); break; - } default: dprintf("smp_intercpu_int_handler: got unknown message %" B_PRId32 "\n", diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index a2cebdbb37..3aee21c3dd 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2681,7 +2681,6 @@ thread_init(kernel_args *args) } gCPU[i].running_thread = thread; - thread->previous_cpu = &gCPU[i]; thread->team = team_get_kernel_team(); thread->priority = B_IDLE_PRIORITY; @@ -2698,6 +2697,8 @@ thread_init(kernel_args *args) thread->visible = true; insert_thread_into_team(thread->team, thread); + + scheduler_on_thread_init(thread); } sUsedThreads = args->num_cpus; From 6a164daad43169daf0a407fb1f6b0484671ff466 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 18 Nov 2013 01:17:44 +0100 Subject: [PATCH 118/273] kernel: Track load produced by interrupt handlers --- headers/private/kernel/int.h | 13 +++- headers/private/kernel/load_tracking.h | 55 ++++++++++++++++ src/system/kernel/arch/x86/32/descriptors.cpp | 2 +- src/system/kernel/arch/x86/arch_smp.cpp | 3 +- src/system/kernel/arch/x86/ioapic.cpp | 2 +- src/system/kernel/arch/x86/pic.cpp | 2 +- .../kernel/arch/x86/timers/x86_apic.cpp | 3 +- src/system/kernel/int.cpp | 63 ++++++++++++++++++- src/system/kernel/scheduler/scheduler.cpp | 52 ++------------- 9 files changed, 140 insertions(+), 55 deletions(-) create mode 100644 headers/private/kernel/load_tracking.h diff --git a/headers/private/kernel/int.h b/headers/private/kernel/int.h index b3642c4d80..215deb03dc 100644 --- a/headers/private/kernel/int.h +++ b/headers/private/kernel/int.h @@ -19,6 +19,16 @@ struct kernel_args; +enum interrupt_type { + INTERRUPT_TYPE_EXCEPTION, + INTERRUPT_TYPE_IRQ, + INTERRUPT_TYPE_LOCAL_IRQ, + INTERRUPT_TYPE_SYSCALL, + INTERRUPT_TYPE_ICI, + INTERRUPT_TYPE_UNKNOWN +}; + + #ifdef __cplusplus extern "C" { #endif @@ -53,7 +63,8 @@ are_interrupts_enabled(void) #define restore_interrupts(status) arch_int_restore_interrupts(status) -status_t reserve_io_interrupt_vectors(long count, long startVector); +status_t reserve_io_interrupt_vectors(long count, long startVector, + enum interrupt_type type); status_t allocate_io_interrupt_vectors(long count, long *startVector); void free_io_interrupt_vectors(long count, long startVector); diff --git a/headers/private/kernel/load_tracking.h b/headers/private/kernel/load_tracking.h new file mode 100644 index 0000000000..5b45c29b9c --- /dev/null +++ b/headers/private/kernel/load_tracking.h @@ -0,0 +1,55 @@ +/* + * Copyright 2013 Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_LOAD_TRACKING_H +#define _KERNEL_LOAD_TRACKING_H + + +const int32 kMaxLoad = 1000; +const bigtime_t kLoadMeasureInterval = 50000; +const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; + + +static int32 +compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load) +{ + bigtime_t now = system_time(); + + if (measureTime == 0) { + measureTime = now; + return -1; + } + + bigtime_t deltaTime = now - measureTime; + + if (deltaTime < kLoadMeasureInterval) + return -1; + + int32 oldLoad = load; + ASSERT(oldLoad >= 0 && oldLoad <= kMaxLoad); + + int32 newLoad = measureActiveTime * kMaxLoad; + newLoad /= max_c(deltaTime, 1); + newLoad = max_c(min_c(newLoad, kMaxLoad), 0); + + measureActiveTime = 0; + measureTime = now; + + deltaTime += kIntervalInaccuracy; + int n = deltaTime / kLoadMeasureInterval; + ASSERT(n > 0); + + if (n > 10) + load = newLoad; + else { + newLoad *= (1 << n) - 1; + load = (load + newLoad) / (1 << n); + ASSERT(load >= 0 && load <= kMaxLoad); + } + + return oldLoad; +} + + +#endif // _KERNEL_LOAD_TRACKING_H diff --git a/src/system/kernel/arch/x86/32/descriptors.cpp b/src/system/kernel/arch/x86/32/descriptors.cpp index 465aab78ca..d215dc0ffb 100644 --- a/src/system/kernel/arch/x86/32/descriptors.cpp +++ b/src/system/kernel/arch/x86/32/descriptors.cpp @@ -311,7 +311,7 @@ x86_descriptors_init(kernel_args* args) set_trap_gate(0, 98, &trap98); // for performance testing only set_trap_gate(0, 99, &trap99); // syscall interrupt - reserve_io_interrupt_vectors(2, 98); + reserve_io_interrupt_vectors(2, 98, INTERRUPT_TYPE_SYSCALL); // configurable msi or msi-x interrupts set_interrupt_gate(0, 100, &trap100); diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index 8c2bc75326..f9a56ba6c3 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -90,7 +90,8 @@ arch_smp_init(kernel_args *args) if (args->num_cpus > 1) { // I/O interrupts start at ARCH_INTERRUPT_BASE, so all interrupts are shifted - reserve_io_interrupt_vectors(3, 0xfd - ARCH_INTERRUPT_BASE); + reserve_io_interrupt_vectors(3, 0xfd - ARCH_INTERRUPT_BASE, + INTERRUPT_TYPE_ICI); install_io_interrupt_handler(0xfd - ARCH_INTERRUPT_BASE, &x86_ici_interrupt, NULL, B_NO_LOCK_VECTOR); install_io_interrupt_handler(0xfe - ARCH_INTERRUPT_BASE, &x86_smp_error_interrupt, NULL, B_NO_LOCK_VECTOR); install_io_interrupt_handler(0xff - ARCH_INTERRUPT_BASE, &x86_spurious_interrupt, NULL, B_NO_LOCK_VECTOR); diff --git a/src/system/kernel/arch/x86/ioapic.cpp b/src/system/kernel/arch/x86/ioapic.cpp index 28d5ba871b..87dd94c48c 100644 --- a/src/system/kernel/arch/x86/ioapic.cpp +++ b/src/system/kernel/arch/x86/ioapic.cpp @@ -774,7 +774,7 @@ ioapic_init(kernel_args* args) current = sIOAPICs; while (current != NULL) { reserve_io_interrupt_vectors(current->max_redirection_entry + 1, - current->global_interrupt_base); + current->global_interrupt_base, INTERRUPT_TYPE_IRQ); current = current->next; } diff --git a/src/system/kernel/arch/x86/pic.cpp b/src/system/kernel/arch/x86/pic.cpp index 7e086312c2..96cd171d15 100644 --- a/src/system/kernel/arch/x86/pic.cpp +++ b/src/system/kernel/arch/x86/pic.cpp @@ -231,7 +231,7 @@ pic_init() TRACE(("PIC level trigger mode: 0x%08lx\n", sLevelTriggeredInterrupts)); - reserve_io_interrupt_vectors(16, 0); + reserve_io_interrupt_vectors(16, 0, INTERRUPT_TYPE_EXCEPTION); // make the pic controller the current one arch_int_set_interrupt_controller(picController); diff --git a/src/system/kernel/arch/x86/timers/x86_apic.cpp b/src/system/kernel/arch/x86/timers/x86_apic.cpp index 3569e75aba..6ac3e5947f 100644 --- a/src/system/kernel/arch/x86/timers/x86_apic.cpp +++ b/src/system/kernel/arch/x86/timers/x86_apic.cpp @@ -106,7 +106,8 @@ apic_timer_init(struct kernel_args *args) sApicTicsPerSec = args->arch_args.apic_time_cv_factor; - reserve_io_interrupt_vectors(1, 0xfb - ARCH_INTERRUPT_BASE); + reserve_io_interrupt_vectors(1, 0xfb - ARCH_INTERRUPT_BASE, + INTERRUPT_TYPE_LOCAL_IRQ); install_io_interrupt_handler(0xfb - ARCH_INTERRUPT_BASE, &apic_timer_interrupt, NULL, B_NO_LOCK_VECTOR); diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 6f9de63a76..df35b67c17 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,13 @@ struct io_vector { spinlock vector_lock; int32 enable_count; bool no_lock_vector; + interrupt_type type; + + spinlock load_lock; + bigtime_t last_measure_time; + bigtime_t last_measure_active; + int32 load; + #if DEBUG_INTERRUPTS int64 handled_count; int64 unhandled_count; @@ -116,6 +124,29 @@ dump_int_statistics(int argc, char **argv) #endif +static int +dump_int_load(int argc, char** argv) +{ + static const char* typeNames[] + = { "exception", "irq", "local irq", "syscall", "ici", "unknown" }; + + for (int i = 0; i < NUM_IO_VECTORS; i++) { + if (!B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) + && sVectors[i].handler_list == NULL + && sVectors[i].enable_count == 0) + continue; + + kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32 + "%%%s\n", i, typeNames[min_c(sVectors[i].type, + INTERRUPT_TYPE_UNKNOWN)], + sVectors[i].enable_count, sVectors[i].load / 10, + B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) ? ", ACTIVE" : ""); + } + + return 0; +} + + // #pragma mark - private kernel API @@ -145,6 +176,13 @@ int_init_post_vm(kernel_args* args) B_INITIALIZE_SPINLOCK(&sVectors[i].vector_lock); sVectors[i].enable_count = 0; sVectors[i].no_lock_vector = false; + sVectors[i].type = INTERRUPT_TYPE_UNKNOWN; + + B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock); + sVectors[i].last_measure_time = 0; + sVectors[i].last_measure_active = 0; + sVectors[i].load = 0; + #if DEBUG_INTERRUPTS sVectors[i].handled_count = 0; sVectors[i].unhandled_count = 0; @@ -159,6 +197,9 @@ int_init_post_vm(kernel_args* args) "list interrupt statistics"); #endif + add_debugger_command("int_load", &dump_int_load, + "list interrupt usage statistics"); + return arch_int_init_post_vm(args); } @@ -179,6 +220,17 @@ int_init_post_device_manager(kernel_args* args) } +static void +update_int_load(int i) +{ + if (!try_acquire_spinlock(&sVectors[i].load_lock)) + return; + compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active, + sVectors[i].load); + release_spinlock(&sVectors[i].load_lock); +} + + /*! Actually process an interrupt via the handlers registered for that vector (IRQ). */ @@ -189,6 +241,8 @@ int_io_interrupt_handler(int vector, bool levelTriggered) struct io_handler* io; bool handled = false; + bigtime_t start = system_time(); + if (!sVectors[vector].no_lock_vector) acquire_spinlock(&sVectors[vector].vector_lock); @@ -267,6 +321,12 @@ int_io_interrupt_handler(int vector, bool levelTriggered) if (!sVectors[vector].no_lock_vector) release_spinlock(&sVectors[vector].vector_lock); + SpinLocker locker(sVectors[vector].load_lock); + sVectors[vector].last_measure_active += system_time() - start; + locker.Unlock(); + + update_int_load(vector); + if (levelTriggered) return status; @@ -435,7 +495,7 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data) vectors using allocate_io_interrupt_vectors() instead. */ status_t -reserve_io_interrupt_vectors(long count, long startVector) +reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type) { MutexLocker locker(&sIOInterruptVectorAllocationLock); @@ -448,6 +508,7 @@ reserve_io_interrupt_vectors(long count, long startVector) return B_BUSY; } + sVectors[startVector + i].type = type; sAllocatedIOInterruptVectors[startVector + i] = true; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index d8a3d98f8d..86760d5bd5 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -61,10 +62,9 @@ const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; const bigtime_t kCacheExpire = 100000; -const int kTargetLoad = 550; -const int kHighLoad = 700; -const int kMaxLoad = 1000; -const int kLoadDifference = 200; +const int kTargetLoad = kMaxLoad * 55 / 100; +const int kHighLoad = kMaxLoad * 70 / 100; +const int kLoadDifference = kMaxLoad * 20 / 100; static bigtime_t sDisableSmallTaskPacking; static int32 sSmallTaskCore; @@ -924,50 +924,6 @@ should_rebalance(Thread* thread) } -static inline int -compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load) -{ - const bigtime_t kLoadMeasureInterval = 50000; - const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; - - bigtime_t now = system_time(); - - if (measureTime == 0) { - measureTime = now; - return -1; - } - - bigtime_t deltaTime = now - measureTime; - - if (deltaTime < kLoadMeasureInterval) - return -1; - - int oldLoad = load; - ASSERT(oldLoad >= 0 && oldLoad <= kMaxLoad); - - int newLoad = measureActiveTime * kMaxLoad; - newLoad /= max_c(deltaTime, 1); - newLoad = max_c(min_c(newLoad, kMaxLoad), 0); - - measureActiveTime = 0; - measureTime = now; - - deltaTime += kIntervalInaccuracy; - int n = deltaTime / kLoadMeasureInterval; - ASSERT(n > 0); - - if (n > 10) - load = newLoad; - else { - newLoad *= (1 << n) - 1; - load = (load + newLoad) / (1 << n); - ASSERT(load >= 0 && load <= kMaxLoad); - } - - return oldLoad; -} - - static inline void compute_cpu_load(int32 cpu) { From 955c7edec2e0d6f52e5245ce449804cede5de34b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 18 Nov 2013 01:50:37 +0100 Subject: [PATCH 119/273] kernel: Measure time spent in interrupt handlers --- headers/private/kernel/cpu.h | 2 ++ src/system/kernel/int.cpp | 10 +++++++++- src/system/kernel/system_info.cpp | 6 ++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index c1524b1d16..7d1dcd365a 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -60,6 +60,8 @@ typedef struct cpu_ent { // keeping track of CPU activity bigtime_t active_time; + bigtime_t irq_time; + bigtime_t interrupt_time; bigtime_t last_kernel_time; bigtime_t last_user_time; diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index df35b67c17..6e1b1fd446 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -243,6 +243,10 @@ int_io_interrupt_handler(int vector, bool levelTriggered) bigtime_t start = system_time(); + // exceptions and syscalls have their own handlers + ASSERT(sVectors[vector].type != INTERRUPT_TYPE_EXCEPTION + && sVectors[vector].type != INTERRUPT_TYPE_SYSCALL); + if (!sVectors[vector].no_lock_vector) acquire_spinlock(&sVectors[vector].vector_lock); @@ -322,9 +326,13 @@ int_io_interrupt_handler(int vector, bool levelTriggered) release_spinlock(&sVectors[vector].vector_lock); SpinLocker locker(sVectors[vector].load_lock); - sVectors[vector].last_measure_active += system_time() - start; + bigtime_t deltaTime = system_time() - start; + sVectors[vector].last_measure_active += deltaTime; locker.Unlock(); + atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime); + if (sVectors[vector].type == INTERRUPT_TYPE_IRQ) + atomic_add64(&get_cpu_struct()->irq_time, deltaTime); update_int_load(vector); if (levelTriggered) diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index a97cdcad10..d6a9fab733 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -50,10 +50,12 @@ dump_info(int argc, char **argv) __VERSION__); kprintf("revision: %s\n\n", get_haiku_revision()); - kprintf("cpu count: %" B_PRId32 ", active times:\n", smp_get_num_cpus()); + kprintf("cpu count: %" B_PRId32 "\n", smp_get_num_cpus()); for (int32 i = 0; i < smp_get_num_cpus(); i++) - kprintf(" [%" B_PRId32 "] %" B_PRId64 "\n", i + 1, gCPU[i].active_time); + kprintf(" [%" B_PRId32 "] active time: %12" B_PRId64 ", interrupt" + " time: %12" B_PRId64 ", irq time: %12" B_PRId64 "\n", i + 1, + gCPU[i].active_time, gCPU[i].interrupt_time, gCPU[i].irq_time); // ToDo: Add page_faults kprintf("pages:\t\t%" B_PRIuPHYSADDR " (%" B_PRIuPHYSADDR " max)\n", From d897a478d7c01054aad29b23f7f545073c797530 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 18 Nov 2013 04:55:25 +0100 Subject: [PATCH 120/273] kernel: Allow reassigning IRQs to logical processors --- headers/private/kernel/arch/int.h | 1 + headers/private/kernel/arch/x86/arch_int.h | 1 + headers/private/kernel/arch/x86/arch_smp.h | 14 +++ headers/private/kernel/cpu.h | 7 +- headers/private/kernel/int.h | 16 +++ src/system/kernel/arch/x86/arch_int.cpp | 8 ++ src/system/kernel/arch/x86/arch_smp.cpp | 8 ++ src/system/kernel/arch/x86/ioapic.cpp | 46 ++++++-- src/system/kernel/arch/x86/pic.cpp | 3 +- src/system/kernel/cpu.cpp | 3 + src/system/kernel/int.cpp | 117 ++++++++++++++++++--- src/system/kernel/system_info.cpp | 4 +- 12 files changed, 197 insertions(+), 31 deletions(-) diff --git a/headers/private/kernel/arch/int.h b/headers/private/kernel/arch/int.h index 9ebe6b0717..6d4afc6f67 100644 --- a/headers/private/kernel/arch/int.h +++ b/headers/private/kernel/arch/int.h @@ -34,6 +34,7 @@ void arch_int_enable_io_interrupt(int irq); void arch_int_disable_io_interrupt(int irq); void arch_int_configure_io_interrupt(int irq, uint32 config); bool arch_int_are_interrupts_enabled(void); +void arch_int_assign_to_cpu(int32 irq, int32 cpu); #ifdef __cplusplus } diff --git a/headers/private/kernel/arch/x86/arch_int.h b/headers/private/kernel/arch/x86/arch_int.h index da4943726d..5de0dcb9d1 100644 --- a/headers/private/kernel/arch/x86/arch_int.h +++ b/headers/private/kernel/arch/x86/arch_int.h @@ -68,6 +68,7 @@ typedef struct interrupt_controller_s { bool (*is_spurious_interrupt)(int32 num); bool (*is_level_triggered_interrupt)(int32 num); bool (*end_of_interrupt)(int32 num); + void (*assign_interrupt_to_cpu)(int32 num, int32 cpu); } interrupt_controller; diff --git a/headers/private/kernel/arch/x86/arch_smp.h b/headers/private/kernel/arch/x86/arch_smp.h index 5b2d654efc..4eef8eb00f 100644 --- a/headers/private/kernel/arch/x86/arch_smp.h +++ b/headers/private/kernel/arch/x86/arch_smp.h @@ -99,4 +99,18 @@ enum { MP_INTR_TYPE_ExtINT, }; + +#ifdef __cplusplus +extern "C" { +#endif + + +uint32 x86_get_cpu_apic_id(int32 cpu); + + +#ifdef __cplusplus +} +#endif + + #endif /* _KERNEL_ARCH_x86_ARCH_SMP_H */ diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 7d1dcd365a..4a70428350 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -11,6 +11,7 @@ #include +#include #include #include #include @@ -79,8 +80,12 @@ typedef struct cpu_ent { int topology_id[CPU_TOPOLOGY_LEVELS]; int cache_id[CPU_MAX_CACHE_LEVEL]; + // IRQs assigned to this CPU + struct list irqs; + spinlock irqs_lock; + // arch-specific stuff - arch_cpu_info arch; + arch_cpu_info arch; } cpu_ent __attribute__((aligned(64))); diff --git a/headers/private/kernel/int.h b/headers/private/kernel/int.h index 215deb03dc..6922aa0573 100644 --- a/headers/private/kernel/int.h +++ b/headers/private/kernel/int.h @@ -12,6 +12,8 @@ #include #include +#include + // private install_io_interrupt_handler() flags #define B_NO_LOCK_VECTOR 0x100 #define B_NO_HANDLED_INFO 0x200 @@ -28,6 +30,18 @@ enum interrupt_type { INTERRUPT_TYPE_UNKNOWN }; +struct irq_assignment { + list_link link; + uint32 irq; + + spinlock load_lock; + bigtime_t last_measure_time; + bigtime_t last_measure_active; + int32 load; + + int32 cpu; +}; + #ifdef __cplusplus extern "C" { @@ -68,4 +82,6 @@ status_t reserve_io_interrupt_vectors(long count, long startVector, status_t allocate_io_interrupt_vectors(long count, long *startVector); void free_io_interrupt_vectors(long count, long startVector); +void assign_io_interrupt_to_cpu(long vector, int32 cpu); + #endif /* _KERNEL_INT_H */ diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 3591f909e8..abcaf5fa08 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -389,6 +389,14 @@ arch_int_are_interrupts_enabled(void) } +void +arch_int_assign_to_cpu(int32 irq, int32 cpu) +{ + if (sCurrentPIC->assign_interrupt_to_cpu != NULL) + sCurrentPIC->assign_interrupt_to_cpu(irq, cpu); +} + + status_t arch_int_init(kernel_args* args) { diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index f9a56ba6c3..340854bb79 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -70,6 +70,14 @@ x86_smp_error_interrupt(void *data) } +uint32 +x86_get_cpu_apic_id(int32 cpu) +{ + ASSERT(cpu >= 0 && cpu < B_MAX_CPU_COUNT); + return sCPUAPICIds[cpu]; +} + + status_t arch_smp_init(kernel_args *args) { diff --git a/src/system/kernel/arch/x86/ioapic.cpp b/src/system/kernel/arch/x86/ioapic.cpp index 87dd94c48c..3f6c5448f6 100644 --- a/src/system/kernel/arch/x86/ioapic.cpp +++ b/src/system/kernel/arch/x86/ioapic.cpp @@ -18,6 +18,7 @@ #include #include +#include #include // to gain access to the ACPICA types @@ -26,9 +27,9 @@ //#define TRACE_IOAPIC #ifdef TRACE_IOAPIC -# define TRACE(x) dprintf x +# define TRACE(...) dprintf(__VA_ARGS__) #else -# define TRACE(x) ; +# define TRACE(...) (void)0 #endif @@ -242,6 +243,30 @@ ioapic_end_of_interrupt(int32 num) } +static void +ioapic_assign_interrupt_to_cpu(int32 gsi, int32 cpu) +{ + if (gsi < ISA_INTERRUPT_COUNT && sSourceOverrides[gsi] != 0) + gsi = sSourceOverrides[gsi]; + + struct ioapic* ioapic = find_ioapic(gsi); + if (ioapic == NULL) + return; + + uint32 apicid = x86_get_cpu_apic_id(cpu); + + uint8 pin = gsi - ioapic->global_interrupt_base; + TRACE("ioapic_assign_interrupt_to_cpu: gsi %ld (io-apic %u pin %u) to" + " cpu %ld (apic_id %lu)\n", gsi, ioapic->number, pin, cpu, apicid); + + uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2); + entry &= ~(uint64(IO_APIC_DESTINATION_FIELD_MASK) + << IO_APIC_DESTINATION_FIELD_SHIFT); + entry |= uint64(apicid) << IO_APIC_DESTINATION_FIELD_SHIFT; + ioapic_write_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2, entry, false); +} + + static void ioapic_enable_io_interrupt(int32 gsi) { @@ -256,8 +281,8 @@ ioapic_enable_io_interrupt(int32 gsi) return; uint8 pin = gsi - ioapic->global_interrupt_base; - TRACE(("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", - gsi, ioapic->number, pin)); + TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", + gsi, ioapic->number, pin); uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2); entry &= ~IO_APIC_INTERRUPT_MASKED; @@ -273,8 +298,8 @@ ioapic_disable_io_interrupt(int32 gsi) return; uint8 pin = gsi - ioapic->global_interrupt_base; - TRACE(("ioapic_disable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", - gsi, ioapic->number, pin)); + TRACE("ioapic_disable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", + gsi, ioapic->number, pin); uint64 entry = ioapic_read_64(*ioapic, IO_APIC_REDIRECTION_TABLE + pin * 2); entry |= IO_APIC_INTERRUPT_MASKED; @@ -290,8 +315,8 @@ ioapic_configure_io_interrupt(int32 gsi, uint32 config) return; uint8 pin = gsi - ioapic->global_interrupt_base; - TRACE(("ioapic_configure_io_interrupt: gsi %ld -> io-apic %u pin %u; " - "config 0x%08lx\n", gsi, ioapic->number, pin, config)); + TRACE("ioapic_configure_io_interrupt: gsi %ld -> io-apic %u pin %u; " + "config 0x%08lx\n", gsi, ioapic->number, pin, config); ioapic_configure_pin(*ioapic, pin, gsi, config, IO_APIC_DELIVERY_MODE_FIXED); @@ -310,7 +335,7 @@ ioapic_map_ioapic(struct ioapic& ioapic, phys_addr_t physicalAddress) return ioapic.register_area; } - TRACE(("mapped io-apic %u to %p\n", ioapic.number, ioapic.registers)); + TRACE("mapped io-apic %u to %p\n", ioapic.number, ioapic.registers); ioapic.version = ioapic_read_32(ioapic, IO_APIC_VERSION); if (ioapic.version == 0xffffffff) { @@ -650,7 +675,8 @@ ioapic_init(kernel_args* args) &ioapic_configure_io_interrupt, &ioapic_is_spurious_interrupt, &ioapic_is_level_triggered_interrupt, - &ioapic_end_of_interrupt + &ioapic_end_of_interrupt, + &ioapic_assign_interrupt_to_cpu, }; if (args->arch_args.apic == NULL) diff --git a/src/system/kernel/arch/x86/pic.cpp b/src/system/kernel/arch/x86/pic.cpp index 96cd171d15..a802ff32d9 100644 --- a/src/system/kernel/arch/x86/pic.cpp +++ b/src/system/kernel/arch/x86/pic.cpp @@ -196,7 +196,8 @@ pic_init() &pic_configure_io_interrupt, &pic_is_spurious_interrupt, &pic_is_level_triggered_interrupt, - &pic_end_of_interrupt + &pic_end_of_interrupt, + NULL }; // Start initialization sequence for the master and slave PICs diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index b98ad77bdd..433d567c75 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -99,6 +99,9 @@ cpu_preboot_init_percpu(kernel_args *args, int curr_cpu) memset(&gCPU[curr_cpu], 0, sizeof(gCPU[curr_cpu])); gCPU[curr_cpu].cpu_num = curr_cpu; + list_init(&gCPU[curr_cpu].irqs); + B_INITIALIZE_SPINLOCK(&gCPU[curr_cpu].irqs_lock); + return arch_cpu_preboot_init_percpu(args, curr_cpu); } diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 6e1b1fd446..cfea6fe33e 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -1,4 +1,7 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + * Copyright 2011, Michael Lotz, mmlr@mlotz.ch. * Distributed under the terms of the MIT License. * @@ -54,10 +57,7 @@ struct io_vector { bool no_lock_vector; interrupt_type type; - spinlock load_lock; - bigtime_t last_measure_time; - bigtime_t last_measure_active; - int32 load; + irq_assignment assigned_cpu; #if DEBUG_INTERRUPTS int64 handled_count; @@ -67,6 +67,7 @@ struct io_vector { #endif }; +static uint32 sLastCPU; static struct io_vector sVectors[NUM_IO_VECTORS]; static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS]; static mutex sIOInterruptVectorAllocationLock @@ -137,10 +138,20 @@ dump_int_load(int argc, char** argv) continue; kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32 - "%%%s\n", i, typeNames[min_c(sVectors[i].type, + "%%", i, typeNames[min_c(sVectors[i].type, INTERRUPT_TYPE_UNKNOWN)], - sVectors[i].enable_count, sVectors[i].load / 10, - B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock) ? ", ACTIVE" : ""); + sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10); + + if (sVectors[i].type == INTERRUPT_TYPE_IRQ) { + if (sVectors[i].assigned_cpu.cpu != -1) + kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu); + else + kprintf(", cpu -"); + } + + if (B_SPINLOCK_IS_LOCKED(&sVectors[i].vector_lock)) + kprintf(", ACTIVE"); + kprintf("\n"); } return 0; @@ -178,10 +189,15 @@ int_init_post_vm(kernel_args* args) sVectors[i].no_lock_vector = false; sVectors[i].type = INTERRUPT_TYPE_UNKNOWN; - B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock); - sVectors[i].last_measure_time = 0; - sVectors[i].last_measure_active = 0; - sVectors[i].load = 0; + irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu; + assigned_cpu->irq = i; + + B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock); + assigned_cpu->last_measure_time = 0; + assigned_cpu->last_measure_active = 0; + assigned_cpu->load = 0; + + assigned_cpu->cpu = -1; #if DEBUG_INTERRUPTS sVectors[i].handled_count = 0; @@ -223,11 +239,14 @@ int_init_post_device_manager(kernel_args* args) static void update_int_load(int i) { - if (!try_acquire_spinlock(&sVectors[i].load_lock)) + if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock)) return; - compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active, - sVectors[i].load); - release_spinlock(&sVectors[i].load_lock); + + compute_load(sVectors[i].assigned_cpu.last_measure_time, + sVectors[i].assigned_cpu.last_measure_active, + sVectors[i].assigned_cpu.load); + + release_spinlock(&sVectors[i].assigned_cpu.load_lock); } @@ -325,9 +344,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered) if (!sVectors[vector].no_lock_vector) release_spinlock(&sVectors[vector].vector_lock); - SpinLocker locker(sVectors[vector].load_lock); + SpinLocker locker(sVectors[vector].assigned_cpu.load_lock); bigtime_t deltaTime = system_time() - start; - sVectors[vector].last_measure_active += deltaTime; + sVectors[vector].assigned_cpu.last_measure_active += deltaTime; locker.Unlock(); atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime); @@ -402,6 +421,23 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, state = disable_interrupts(); acquire_spinlock(&sVectors[vector].vector_lock); + // Initial attempt to balance IRQs, the scheduler will correct this + // if some cores end up being overloaded. + if (sVectors[vector].type == INTERRUPT_TYPE_IRQ + && sVectors[vector].handler_list == NULL) { + + int32 cpuID = atomic_add(&sLastCPU, 1) % smp_get_num_cpus(); + arch_int_assign_to_cpu(vector, cpuID); + + ASSERT(sVectors[vector].assigned_cpu.cpu == -1); + + sVectors[vector].assigned_cpu.cpu = cpuID; + + cpu_ent* cpu = &gCPU[cpuID]; + SpinLocker _(cpu->irqs_lock); + list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + } + if ((flags & B_NO_HANDLED_INFO) != 0 && sVectors[vector].handler_list != NULL) { // The driver registering this interrupt handler doesn't know @@ -441,6 +477,7 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, sVectors[vector].no_lock_vector = true; release_spinlock(&sVectors[vector].vector_lock); + restore_interrupts(state); return B_OK; @@ -486,6 +523,28 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data) last = io; } + if (sVectors[vector].handler_list == NULL + && sVectors[vector].type == INTERRUPT_TYPE_IRQ) { + + int32 oldCPU; + SpinLocker locker; + cpu_ent* cpu; + + do { + locker.Unlock(); + + oldCPU = sVectors[vector].assigned_cpu.cpu; + + ASSERT(oldCPU != -1); + cpu = &gCPU[oldCPU]; + + locker.SetTo(cpu->irqs_lock, false); + } while (sVectors[vector].assigned_cpu.cpu != oldCPU); + + sVectors[vector].assigned_cpu.cpu = -1; + list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + } + release_spinlock(&sVectors[vector].vector_lock); restore_interrupts(state); @@ -596,3 +655,27 @@ free_io_interrupt_vectors(long count, long startVector) sAllocatedIOInterruptVectors[startVector + i] = false; } } + + +void assign_io_interrupt_to_cpu(long vector, int32 newCPU) +{ + ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ); + + int32 oldCPU = sVectors[vector].assigned_cpu.cpu; + + ASSERT(oldCPU != -1); + cpu_ent* cpu = &gCPU[oldCPU]; + + SpinLocker locker(cpu->irqs_lock); + sVectors[vector].assigned_cpu.cpu = -1; + list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + locker.Unlock(); + + cpu = &gCPU[newCPU]; + locker.SetTo(cpu->irqs_lock, false); + sVectors[vector].assigned_cpu.cpu = newCPU; + arch_int_assign_to_cpu(vector, newCPU); + list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + locker.Unlock(); +} + diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index d6a9fab733..d6713b00c0 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -53,8 +53,8 @@ dump_info(int argc, char **argv) kprintf("cpu count: %" B_PRId32 "\n", smp_get_num_cpus()); for (int32 i = 0; i < smp_get_num_cpus(); i++) - kprintf(" [%" B_PRId32 "] active time: %12" B_PRId64 ", interrupt" - " time: %12" B_PRId64 ", irq time: %12" B_PRId64 "\n", i + 1, + kprintf(" [%" B_PRId32 "] active time: %10" B_PRId64 ", interrupt" + " time: %10" B_PRId64 ", irq time: %10" B_PRId64 "\n", i + 1, gCPU[i].active_time, gCPU[i].interrupt_time, gCPU[i].irq_time); // ToDo: Add page_faults From f14e4567e81270bf6ddf1a3abfef662885763e3d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 18 Nov 2013 05:32:51 +0100 Subject: [PATCH 121/273] kernel: Use CPU topology to distribute IRQs --- src/system/kernel/int.cpp | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index cfea6fe33e..baed8fd73d 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -67,7 +67,8 @@ struct io_vector { #endif }; -static uint32 sLastCPU; +static int32 sLastCPU; + static struct io_vector sVectors[NUM_IO_VECTORS]; static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS]; static mutex sIOInterruptVectorAllocationLock @@ -387,6 +388,22 @@ restore_interrupts(cpu_status status) } +static +uint32 assign_cpu(void) +{ + int32 nextID = atomic_add(&sLastCPU, 1); + cpu_topology_node* node = get_cpu_topology(); + + while (node->level != CPU_TOPOLOGY_SMT) { + int levelSize = node->children_count; + node = node->children[nextID % levelSize]; + nextID /= levelSize; + } + + return node->id; +} + + /*! Install a handler to be called when an interrupt is triggered for the given interrupt number with \a data as the argument. */ @@ -426,7 +443,7 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, if (sVectors[vector].type == INTERRUPT_TYPE_IRQ && sVectors[vector].handler_list == NULL) { - int32 cpuID = atomic_add(&sLastCPU, 1) % smp_get_num_cpus(); + int32 cpuID = assign_cpu(); arch_int_assign_to_cpu(vector, cpuID); ASSERT(sVectors[vector].assigned_cpu.cpu == -1); From e2ff9a2865beeabed2dbe7cc2b975b377cd12087 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 18 Nov 2013 07:05:35 +0100 Subject: [PATCH 122/273] scheduler: Rebalance IRQs on overloaded cores --- src/system/kernel/scheduler/scheduler.cpp | 53 ++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 86760d5bd5..e8cbffbc9a 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -64,7 +64,9 @@ const bigtime_t kCacheExpire = 100000; const int kTargetLoad = kMaxLoad * 55 / 100; const int kHighLoad = kMaxLoad * 70 / 100; +const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; const int kLoadDifference = kMaxLoad * 20 / 100; +const int kLowLoad = kLoadDifference / 2; static bigtime_t sDisableSmallTaskPacking; static int32 sSmallTaskCore; @@ -76,6 +78,7 @@ static rw_spinlock sSchedulerModeLock = B_RW_SPINLOCK_INITIALIZER; static int32 (*sChooseCore)(Thread* thread); static bool (*sShouldRebalance)(Thread* thread); +static void (*sRebalanceIRQs)(void); // Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical @@ -893,7 +896,7 @@ should_rebalance_power_saving(Thread* thread) if (coreEntry->fLoad > kHighLoad) { if (!is_task_small(thread)) return true; - } else if (coreEntry->fLoad > (kHighLoad + kMaxLoad) / 2) + } else if (coreEntry->fLoad > kVeryHighLoad) disable_small_task_packing(); } @@ -924,6 +927,48 @@ should_rebalance(Thread* thread) } +static void +rebalance_irqs_low_latency(void) +{ + cpu_ent* cpu = get_cpu_struct(); + SpinLocker locker(cpu->irqs_lock); + + irq_assignment* chosen = NULL; + irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); + + int32 totalLoad = 0; + while (irq != NULL) { + if (chosen == NULL || chosen->load < irq->load) + chosen = irq; + totalLoad += irq->load; + irq = (irq_assignment*)list_get_next_item(&cpu->irqs, irq); + } + + locker.Unlock(); + + if (chosen == NULL || totalLoad < kLowLoad) + return; + + SpinLocker coreLocker(sCoreHeapsLock); + CoreEntry* other = sCoreLoadHeap->PeekMinimum(); + if (other == NULL) + other = sCoreHighLoadHeap->PeekMinimum(); + coreLocker.Unlock(); + + ASSERT(other != NULL); + + int32 thisCore = sCPUToCore[smp_get_current_cpu()]; + if (other->fCoreID == thisCore) + return; + + if (other->fLoad + kLoadDifference >= sCoreEntries[thisCore].fLoad) + return; + + int32 newCPU = choose_cpu(other->fCoreID); + assign_io_interrupt_to_cpu(chosen->irq, newCPU); +} + + static inline void compute_cpu_load(int32 cpu) { @@ -934,6 +979,9 @@ compute_cpu_load(int32 cpu) if (oldLoad < 0) return; + if (sCPUEntries[cpu].fLoad > kVeryHighLoad) + sRebalanceIRQs(); + if (oldLoad != sCPUEntries[cpu].fLoad) { int32 core = sCPUToCore[cpu]; @@ -1625,13 +1673,16 @@ scheduler_set_operation_mode(scheduler_mode mode) case SCHEDULER_MODE_LOW_LATENCY: sDisableSmallTaskPacking = -1; sSmallTaskCore = -1; + sChooseCore = choose_core_low_latency; sShouldRebalance = should_rebalance_low_latency; + sRebalanceIRQs = rebalance_irqs_low_latency; break; case SCHEDULER_MODE_POWER_SAVING: sDisableSmallTaskPacking = 0; sSmallTaskCore = -1; + sChooseCore = choose_core_power_saving; sShouldRebalance = should_rebalance_power_saving; break; From 9c2e74da043b32ecd64a0ec4f4917f83ac6cc10f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 09:46:59 +0100 Subject: [PATCH 123/273] scheduler: Move mode specific logic to separate files --- headers/private/kernel/kscheduler.h | 1 + headers/private/kernel/load_tracking.h | 5 +- src/system/kernel/Jamfile | 2 + src/system/kernel/scheduler/low_latency.cpp | 154 ++++ src/system/kernel/scheduler/power_saving.cpp | 164 ++++ src/system/kernel/scheduler/scheduler.cpp | 831 +++++------------- .../kernel/scheduler/scheduler_common.h | 159 ++++ src/system/kernel/scheduler/scheduler_modes.h | 29 + 8 files changed, 747 insertions(+), 598 deletions(-) create mode 100644 src/system/kernel/scheduler/low_latency.cpp create mode 100644 src/system/kernel/scheduler/power_saving.cpp create mode 100644 src/system/kernel/scheduler/scheduler_modes.h diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index addd4583cd..dc2bb88093 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2005-2010, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. diff --git a/headers/private/kernel/load_tracking.h b/headers/private/kernel/load_tracking.h index 5b45c29b9c..934bcc79de 100644 --- a/headers/private/kernel/load_tracking.h +++ b/headers/private/kernel/load_tracking.h @@ -6,12 +6,15 @@ #define _KERNEL_LOAD_TRACKING_H +#include + + const int32 kMaxLoad = 1000; const bigtime_t kLoadMeasureInterval = 50000; const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; -static int32 +static inline int32 compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load) { bigtime_t now = system_time(); diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 9bf77eb834..4173486dff 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -62,6 +62,8 @@ KernelMergeObject kernel_core.o : user_mutex.cpp # scheduler + low_latency.cpp + power_saving.cpp scheduler.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp new file mode 100644 index 0000000000..7a0ee48570 --- /dev/null +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -0,0 +1,154 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include "scheduler_common.h" +#include "scheduler_modes.h" + + +using namespace Scheduler; + + +static void +switch_to_mode(void) +{ +} + + +static bool +has_cache_expired(Thread* thread) +{ + ASSERT(!gSingleCore); + + if (thread_is_idle_thread(thread)) + return false; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; + return atomic_get64(&coreEntry->fActiveTime) + - schedulerThreadData->went_sleep_active > kCacheExpire; +} + + +static int32 +choose_core(Thread* thread) +{ + CoreEntry* entry; + + if (gIdlePackageList->Last() != NULL) { + // wake new package + PackageEntry* package = gIdlePackageList->Last(); + entry = package->fIdleCores.Last(); + } else if (gPackageUsageHeap->PeekMaximum() != NULL) { + // wake new core + PackageEntry* package = gPackageUsageHeap->PeekMaximum(); + entry = package->fIdleCores.Last(); + } else { + // no idle cores, use least occupied core + entry = gCoreLoadHeap->PeekMinimum(); + if (entry == NULL) + entry = gCoreHighLoadHeap->PeekMinimum(); + } + + ASSERT(entry != NULL); + return entry->fCoreID; +} + + +static bool +should_rebalance(Thread* thread) +{ + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; + + // If the thread produces more than 50% of the load, leave it here. In + // such situation it is better to move other threads away. + if (schedulerThreadData->load >= coreEntry->fLoad / 2) + return false; + + // If there is high load on this core but this thread does not contribute + // significantly consider giving it to someone less busy. + if (coreEntry->fLoad > kHighLoad) { + SpinLocker coreLocker(gCoreHeapsLock); + + CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference) + return true; + } + + // No cpu bound threads - the situation is quite good. Make sure it + // won't get much worse... + SpinLocker coreLocker(gCoreHeapsLock); + + CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + if (other == NULL) + other = gCoreHighLoadHeap->PeekMinimum(); + return coreEntry->fLoad - other->fLoad >= kLoadDifference * 2; +} + + +static void +rebalance_irqs(bool idle) +{ + if (idle) + return; + + cpu_ent* cpu = get_cpu_struct(); + SpinLocker locker(cpu->irqs_lock); + + irq_assignment* chosen = NULL; + irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); + + int32 totalLoad = 0; + while (irq != NULL) { + if (chosen == NULL || chosen->load < irq->load) + chosen = irq; + totalLoad += irq->load; + irq = (irq_assignment*)list_get_next_item(&cpu->irqs, irq); + } + + locker.Unlock(); + + if (chosen == NULL || totalLoad < kLowLoad) + return; + + SpinLocker coreLocker(gCoreHeapsLock); + CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + if (other == NULL) + other = gCoreHighLoadHeap->PeekMinimum(); + coreLocker.Unlock(); + + ASSERT(other != NULL); + + int32 thigCore = gCPUToCore[smp_get_current_cpu()]; + if (other->fCoreID == thigCore) + return; + + if (other->fLoad + kLoadDifference >= gCoreEntries[thigCore].fLoad) + return; + + coreLocker.Lock(); + gCPUPriorityHeaps[other->fCoreID].PeekMinimum(); +} + + +scheduler_mode_operations gSchedulerLowLatencyMode = { + "low latency", + + true, + + switch_to_mode, + has_cache_expired, + choose_core, + should_rebalance, + rebalance_irqs, +}; + diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp new file mode 100644 index 0000000000..aed4e5fd23 --- /dev/null +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -0,0 +1,164 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include "scheduler_common.h" +#include "scheduler_modes.h" + + +using namespace Scheduler; + + +static bigtime_t sDisableSmallTaskPacking; +static int32 sSmallTaskCore; + + +static bool +has_cache_expired(Thread* thread) +{ + ASSERT(!gSingleCore); + + if (thread_is_idle_thread(thread)) + return false; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; + return system_time() - schedulerThreadData->went_sleep > kCacheExpire; +} + + +static inline bool +is_small_task_packing_enabled(void) +{ + if (sDisableSmallTaskPacking == -1) + return false; + return sDisableSmallTaskPacking < system_time(); +} + + +static inline void +disable_small_task_packing(void) +{ + ASSERT(!gSingleCore); + + ASSERT(is_small_task_packing_enabled()); + ASSERT(sSmallTaskCore == gCPUToCore[smp_get_current_cpu()]); + + sDisableSmallTaskPacking = system_time() + kThreadQuantum * 100; + sSmallTaskCore = -1; +} + + +static inline bool +is_task_small(Thread* thread) +{ + return thread->scheduler_data->load <= 200; +} + + +static void +switch_to_mode(void) +{ + sDisableSmallTaskPacking = -1; + sSmallTaskCore = -1; +} + + +static int32 +choose_core(Thread* thread) +{ + CoreEntry* entry; + + if (is_small_task_packing_enabled() && is_task_small(thread) + && gCoreLoadHeap->PeekMaximum() != NULL) { + // try to pack all threads on one core + if (sSmallTaskCore < 0) + sSmallTaskCore = gCoreLoadHeap->PeekMaximum()->fCoreID; + entry = &gCoreEntries[sSmallTaskCore]; + } else if (gCoreLoadHeap->PeekMinimum() != NULL) { + // run immediately on already woken core + entry = gCoreLoadHeap->PeekMinimum(); + } else if (gPackageUsageHeap->PeekMinimum() != NULL) { + // wake new core + PackageEntry* package = gPackageUsageHeap->PeekMinimum(); + entry = package->fIdleCores.Last(); + } else if (gIdlePackageList->Last() != NULL) { + // wake new package + PackageEntry* package = gIdlePackageList->Last(); + entry = package->fIdleCores.Last(); + } else { + // no idle cores, use least occupied core + entry = gCoreLoadHeap->PeekMinimum(); + if (entry == NULL) + entry = gCoreHighLoadHeap->PeekMinimum(); + } + + ASSERT(entry != NULL); + return entry->fCoreID; +} + + +static bool +should_rebalance(Thread* thread) +{ + ASSERT(!gSingleCore); + + if (thread_is_idle_thread(thread)) + return false; + + scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ASSERT(schedulerThreadData->previous_core >= 0); + + int32 core = schedulerThreadData->previous_core; + CoreEntry* coreEntry = &gCoreEntries[core]; + + // If the thread produces more than 50% of the load, leave it here. In + // such situation it is better to move other threads away. + // Unless we are trying to pack small tasks here, in such case get rid + // of CPU hungry thread and continue packing. + if (schedulerThreadData->load >= coreEntry->fLoad / 2) + return is_small_task_packing_enabled() && sSmallTaskCore == core; + + // All cores try to give us small tasks, check whether we have enough. + if (is_small_task_packing_enabled() && sSmallTaskCore == core) { + if (coreEntry->fLoad > kHighLoad) { + if (!is_task_small(thread)) + return true; + } else if (coreEntry->fLoad > kVeryHighLoad) + disable_small_task_packing(); + } + + // Try small task packing. + if (is_small_task_packing_enabled() && is_task_small(thread)) + return sSmallTaskCore != core; + + // No cpu bound threads - the situation is quite good. Make sure it + // won't get much worse... + SpinLocker coreLocker(gCoreHeapsLock); + + CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + if (other == NULL) + other = gCoreHighLoadHeap->PeekMinimum(); + return coreEntry->fLoad - other->fLoad >= kLoadDifference; +} + + +scheduler_mode_operations gSchedulerPowerSavingMode = { + "power saving", + + false, + + switch_to_mode, + has_cache_expired, + choose_core, + should_rebalance, + NULL, +}; + + diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index e8cbffbc9a..394706ef47 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -26,177 +26,56 @@ #include #include #include -#include #include -#include -#include #include -#include "RunQueue.h" #include "scheduler_common.h" +#include "scheduler_modes.h" #include "scheduler_tracing.h" -//#define TRACE_SCHEDULER -#ifdef TRACE_SCHEDULER -# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) -#else -# define TRACE(...) do { } while (false) -#endif +using namespace Scheduler; -#define CACHE_LINE_ALIGN __attribute__((aligned(64))) - +static bool sSchedulerEnabled; SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -static bool sSchedulerEnabled; - -const bigtime_t kThreadQuantum = 1000; -const bigtime_t kMinThreadQuantum = 3000; -const bigtime_t kMaxThreadQuantum = 10000; - -const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; - -const bigtime_t kCacheExpire = 100000; - -const int kTargetLoad = kMaxLoad * 55 / 100; -const int kHighLoad = kMaxLoad * 70 / 100; -const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; -const int kLoadDifference = kMaxLoad * 20 / 100; -const int kLowLoad = kLoadDifference / 2; - -static bigtime_t sDisableSmallTaskPacking; -static int32 sSmallTaskCore; - -static bool sSingleCore; - -static scheduler_mode sSchedulerMode; static rw_spinlock sSchedulerModeLock = B_RW_SPINLOCK_INITIALIZER; - -static int32 (*sChooseCore)(Thread* thread); -static bool (*sShouldRebalance)(Thread* thread); -static void (*sRebalanceIRQs)(void); - - -// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical -// processors in the heap belong to. Since there are no cache affinity issues -// at this level and the run queue is shared among all logical processors on -// the core the only real concern is to make lower priority threads give way to -// the higher priority threads. -struct CPUEntry : public MinMaxHeapLinkImpl { - CPUEntry(); - - int32 fCPUNumber; - - int32 fPriority; - - bigtime_t fMeasureActiveTime; - bigtime_t fMeasureTime; - - int32 fLoad; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; - -static CPUEntry* sCPUEntries; -static CPUHeap* sCPUPriorityHeaps; - -struct CoreEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { - CoreEntry(); - - int32 fCoreID; - - spinlock fLock; - - bigtime_t fStartedBottom; - bigtime_t fReachedBottom; - bigtime_t fStartedIdle; - bigtime_t fReachedIdle; - - bigtime_t fActiveTime; - - int32 fLoad; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap CoreLoadHeap; - -static CoreEntry* sCoreEntries; -static CoreLoadHeap* sCoreLoadHeap; -static CoreLoadHeap* sCoreHighLoadHeap; -static spinlock sCoreHeapsLock = B_SPINLOCK_INITIALIZER; - -// sPackageUsageHeap is used to decide which core should be woken up from the -// idle state. When aiming for performance we should use as many packages as -// possible with as little cores active in each package as possible (so that the -// package can enter any boost mode if it has one and the active core have more -// of the shared cache for themselves. If power saving is the main priority we -// should keep active cores on as little packages as possible (so that other -// packages can go to the deep state of sleep). The heap stores only packages -// with at least one core active and one core idle. The packages with all cores -// idle are stored in sPackageIdleList (in LIFO manner). -struct PackageEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { - PackageEntry(); - - int32 fPackageID; - - DoublyLinkedList fIdleCores; - int32 fIdleCoreCount; - - int32 fCoreCount; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap PackageHeap; -typedef DoublyLinkedList IdlePackageList; - -static PackageEntry* sPackageEntries; -static PackageHeap* sPackageUsageHeap; -static IdlePackageList* sIdlePackageList; -static spinlock sIdlePackageLock = B_SPINLOCK_INITIALIZER; - -// The run queues. Holds the threads ready to run ordered by priority. -// One queue per schedulable target per core. Additionally, each -// logical processor has its sPinnedRunQueues used for scheduling -// pinned threads. -typedef RunQueue ThreadRunQueue; - -static ThreadRunQueue* sRunQueues CACHE_LINE_ALIGN; -static ThreadRunQueue* sPinnedRunQueues CACHE_LINE_ALIGN; -static int32 sRunQueueCount; - -// Since CPU IDs used internally by the kernel bear no relation to the actual -// CPU topology the following arrays are used to efficiently get the core -// and the package that CPU in question belongs to. -static int32* sCPUToCore; -static int32* sCPUToPackage; - -struct scheduler_thread_data { - scheduler_thread_data() { Init(); } - inline void Init(); - - int32 priority_penalty; - int32 additional_penalty; - - bool lost_cpu; - bool cpu_bound; - - bigtime_t time_left; - bigtime_t stolen_time; - bigtime_t quantum_start; - - bigtime_t measure_active_time; - bigtime_t measure_time; - int32 load; - - bigtime_t went_sleep; - bigtime_t went_sleep_active; - - int32 previous_core; - - bool enqueued; +static struct scheduler_mode_operations* sCurrentMode; +static struct scheduler_mode_operations* sSchedulerModes[] = { + &gSchedulerLowLatencyMode, + &gSchedulerPowerSavingMode, }; +namespace Scheduler { + +bool gSingleCore; + +CPUEntry* gCPUEntries; +CPUHeap* gCPUPriorityHeaps; + +CoreEntry* gCoreEntries; +CoreLoadHeap* gCoreLoadHeap; +CoreLoadHeap* gCoreHighLoadHeap; +spinlock gCoreHeapsLock = B_SPINLOCK_INITIALIZER; + +PackageEntry* gPackageEntries; +PackageHeap* gPackageUsageHeap; +IdlePackageList* gIdlePackageList; +spinlock gIdlePackageLock = B_SPINLOCK_INITIALIZER; + +ThreadRunQueue* gRunQueues; +ThreadRunQueue* gPinnedRunQueues; +int32 gRunQueueCount; + +int32* gCPUToCore; +int32* gCPUToPackage; + +} // namespace Scheduler + CPUEntry::CPUEntry() : @@ -225,6 +104,12 @@ PackageEntry::PackageEntry() } +scheduler_thread_data::scheduler_thread_data() +{ + Init(); +} + + void scheduler_thread_data::Init() { @@ -313,18 +198,18 @@ dump_run_queue(int argc, char **argv) int32 coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) - sCPUToCore[i] = coreCount++; + gCPUToCore[i] = coreCount++; } ThreadRunQueue::ConstIterator iterator; for (int32 i = 0; i < coreCount; i++) { kprintf("\nCore %" B_PRId32 " run queue:\n", i); - iterator = sRunQueues[i].GetConstIterator(); + iterator = gRunQueues[i].GetConstIterator(); dump_queue(iterator); } for (int32 i = 0; i < cpuCount; i++) { - iterator = sPinnedRunQueues[i].GetConstIterator(); + iterator = gPinnedRunQueues[i].GetConstIterator(); if (iterator.HasNext()) { kprintf("\nCPU %" B_PRId32 " run queue:\n", i); @@ -347,7 +232,7 @@ dump_heap(CPUHeap* heap) int32 cpu = entry->fCPUNumber; int32 key = CPUHeap::GetKey(entry); kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, - sCPUEntries[cpu].fLoad / 10); + gCPUEntries[cpu].fLoad / 10); heap->RemoveMinimum(); temp.Insert(entry, key); @@ -368,8 +253,8 @@ dump_heap(CPUHeap* heap) static void dump_core_load_heap(CoreLoadHeap* heap) { - CoreLoadHeap temp(sRunQueueCount); - int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; + CoreLoadHeap temp(gRunQueueCount); + int32 cpuPerCore = smp_get_num_cpus() / gRunQueueCount; CoreEntry* entry = heap->PeekMinimum(); while (entry) { @@ -397,13 +282,13 @@ static int dump_cpu_heap(int argc, char** argv) { kprintf("\ncore load\n"); - dump_core_load_heap(sCoreLoadHeap); + dump_core_load_heap(gCoreLoadHeap); kprintf("---------\n"); - dump_core_load_heap(sCoreHighLoadHeap); + dump_core_load_heap(gCoreHighLoadHeap); - for (int32 i = 0; i < sRunQueueCount; i++) { + for (int32 i = 0; i < gRunQueueCount; i++) { kprintf("\nCore %" B_PRId32 " heap:\n", i); - dump_heap(&sCPUPriorityHeaps[i]); + dump_heap(&gCPUPriorityHeaps[i]); } return 0; @@ -415,7 +300,7 @@ dump_idle_cores(int argc, char** argv) { kprintf("Idle packages:\n"); IdlePackageList::ReverseIterator idleIterator - = sIdlePackageList->GetReverseIterator(); + = gIdlePackageList->GetReverseIterator(); if (idleIterator.HasNext()) { kprintf("package cores\n"); @@ -442,7 +327,7 @@ dump_idle_cores(int argc, char** argv) PackageHeap temp(smp_get_num_cpus()); kprintf("\nPackages with idle cores:\n"); - PackageEntry* entry = sPackageUsageHeap->PeekMinimum(); + PackageEntry* entry = gPackageUsageHeap->PeekMinimum(); if (entry == NULL) kprintf("No packages.\n"); else @@ -464,17 +349,17 @@ dump_idle_cores(int argc, char** argv) kprintf("-"); kprintf("\n"); - sPackageUsageHeap->RemoveMinimum(); + gPackageUsageHeap->RemoveMinimum(); temp.Insert(entry, entry->fIdleCoreCount); - entry = sPackageUsageHeap->PeekMinimum(); + entry = gPackageUsageHeap->PeekMinimum(); } entry = temp.PeekMinimum(); while (entry != NULL) { int32 key = PackageHeap::GetKey(entry); temp.RemoveMinimum(); - sPackageUsageHeap->Insert(entry, key); + gPackageUsageHeap->Insert(entry, key); entry = temp.PeekMinimum(); } @@ -485,27 +370,7 @@ dump_idle_cores(int argc, char** argv) static inline bool has_cache_expired(Thread* thread) { - ASSERT(!sSingleCore); - - if (thread_is_idle_thread(thread)) - return false; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; - switch (sSchedulerMode) { - case SCHEDULER_MODE_LOW_LATENCY: - return atomic_get64(&coreEntry->fActiveTime) - - schedulerThreadData->went_sleep_active > kCacheExpire; - - case SCHEDULER_MODE_POWER_SAVING: - return system_time() - schedulerThreadData->went_sleep - > kCacheExpire; - - default: - return true; - } + return sCurrentMode->has_cache_expired(thread); } @@ -544,13 +409,13 @@ scheduler_dump_thread_data(Thread* thread) static void update_load_heaps(int32 core) { - ASSERT(!sSingleCore); + ASSERT(!gSingleCore); - CoreEntry* entry = &sCoreEntries[core]; + CoreEntry* entry = &gCoreEntries[core]; - SpinLocker coreLocker(sCoreHeapsLock); + SpinLocker coreLocker(gCoreHeapsLock); - int32 cpuPerCore = smp_get_num_cpus() / sRunQueueCount; + int32 cpuPerCore = smp_get_num_cpus() / gRunQueueCount; int32 newKey = entry->fLoad / cpuPerCore; int32 oldKey = CoreLoadHeap::GetKey(entry); @@ -562,48 +427,26 @@ update_load_heaps(int32 core) if (newKey > kHighLoad) { if (oldKey <= kHighLoad) { - sCoreLoadHeap->ModifyKey(entry, -1); - ASSERT(sCoreLoadHeap->PeekMinimum() == entry); - sCoreLoadHeap->RemoveMinimum(); + gCoreLoadHeap->ModifyKey(entry, -1); + ASSERT(gCoreLoadHeap->PeekMinimum() == entry); + gCoreLoadHeap->RemoveMinimum(); - sCoreHighLoadHeap->Insert(entry, newKey); + gCoreHighLoadHeap->Insert(entry, newKey); } else - sCoreHighLoadHeap->ModifyKey(entry, newKey); + gCoreHighLoadHeap->ModifyKey(entry, newKey); } else { if (oldKey > kHighLoad) { - sCoreHighLoadHeap->ModifyKey(entry, -1); - ASSERT(sCoreHighLoadHeap->PeekMinimum() == entry); - sCoreHighLoadHeap->RemoveMinimum(); + gCoreHighLoadHeap->ModifyKey(entry, -1); + ASSERT(gCoreHighLoadHeap->PeekMinimum() == entry); + gCoreHighLoadHeap->RemoveMinimum(); - sCoreLoadHeap->Insert(entry, newKey); + gCoreLoadHeap->Insert(entry, newKey); } else - sCoreLoadHeap->ModifyKey(entry, newKey); + gCoreLoadHeap->ModifyKey(entry, newKey); } } -static inline bool -is_small_task_packing_enabled(void) -{ - if (sDisableSmallTaskPacking == -1) - return false; - return sDisableSmallTaskPacking < system_time(); -} - - -static inline void -disable_small_task_packing(void) -{ - ASSERT(!sSingleCore); - - ASSERT(is_small_task_packing_enabled()); - ASSERT(sSmallTaskCore == sCPUToCore[smp_get_current_cpu()]); - - sDisableSmallTaskPacking = system_time() + kThreadQuantum * 100; - sSmallTaskCore = -1; -} - - static inline void increase_penalty(Thread* thread) { @@ -643,162 +486,96 @@ cancel_penalty(Thread* thread) static inline void update_cpu_priority(int32 cpu, int32 priority) { - int32 core = sCPUToCore[cpu]; + int32 core = gCPUToCore[cpu]; - int32 corePriority = CPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); + int32 corePriority = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); - sCPUEntries[cpu].fPriority = priority; - sCPUPriorityHeaps[core].ModifyKey(&sCPUEntries[cpu], priority); + gCPUEntries[cpu].fPriority = priority; + gCPUPriorityHeaps[core].ModifyKey(&gCPUEntries[cpu], priority); - if (sSingleCore) + if (gSingleCore) return; int32 maxPriority - = CPUHeap::GetKey(sCPUPriorityHeaps[core].PeekMaximum()); + = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); if (corePriority == maxPriority) return; - int32 package = sCPUToPackage[cpu]; - PackageEntry* packageEntry = &sPackageEntries[package]; + int32 package = gCPUToPackage[cpu]; + PackageEntry* packageEntry = &gPackageEntries[package]; if (maxPriority == B_IDLE_PRIORITY) { - SpinLocker _(sIdlePackageLock); + SpinLocker _(gIdlePackageLock); // core goes idle ASSERT(packageEntry->fIdleCoreCount >= 0); ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); packageEntry->fIdleCoreCount++; - packageEntry->fIdleCores.Add(&sCoreEntries[core]); + packageEntry->fIdleCores.Add(&gCoreEntries[core]); if (packageEntry->fIdleCoreCount == 1) { // first core on that package to go idle if (packageEntry->fCoreCount > 1) - sPackageUsageHeap->Insert(packageEntry, 1); + gPackageUsageHeap->Insert(packageEntry, 1); else - sIdlePackageList->Add(packageEntry); + gIdlePackageList->Add(packageEntry); } else if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { // package goes idle - sPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); - sPackageUsageHeap->RemoveMinimum(); + gPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(gPackageUsageHeap->PeekMinimum() == packageEntry); + gPackageUsageHeap->RemoveMinimum(); - sIdlePackageList->Add(packageEntry); + gIdlePackageList->Add(packageEntry); } else { - sPackageUsageHeap->ModifyKey(packageEntry, + gPackageUsageHeap->ModifyKey(packageEntry, packageEntry->fIdleCoreCount); } } else if (corePriority == B_IDLE_PRIORITY) { - SpinLocker _(sIdlePackageLock); + SpinLocker _(gIdlePackageLock); // core wakes up ASSERT(packageEntry->fIdleCoreCount > 0); ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); packageEntry->fIdleCoreCount--; - packageEntry->fIdleCores.Remove(&sCoreEntries[core]); + packageEntry->fIdleCores.Remove(&gCoreEntries[core]); if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { // package wakes up - sIdlePackageList->Remove(packageEntry); + gIdlePackageList->Remove(packageEntry); if (packageEntry->fIdleCoreCount > 0) { - sPackageUsageHeap->Insert(packageEntry, + gPackageUsageHeap->Insert(packageEntry, packageEntry->fIdleCoreCount); } } else if (packageEntry->fIdleCoreCount == 0) { // no more idle cores in the package - sPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(sPackageUsageHeap->PeekMinimum() == packageEntry); - sPackageUsageHeap->RemoveMinimum(); + gPackageUsageHeap->ModifyKey(packageEntry, 0); + ASSERT(gPackageUsageHeap->PeekMinimum() == packageEntry); + gPackageUsageHeap->RemoveMinimum(); } else { - sPackageUsageHeap->ModifyKey(packageEntry, + gPackageUsageHeap->ModifyKey(packageEntry, packageEntry->fIdleCoreCount); } } } -static int32 -choose_core_low_latency(Thread* thread) -{ - CoreEntry* entry; - - if (sIdlePackageList->Last() != NULL) { - // wake new package - PackageEntry* package = sIdlePackageList->Last(); - entry = package->fIdleCores.Last(); - } else if (sPackageUsageHeap->PeekMaximum() != NULL) { - // wake new core - PackageEntry* package = sPackageUsageHeap->PeekMaximum(); - entry = package->fIdleCores.Last(); - } else { - // no idle cores, use least occupied core - entry = sCoreLoadHeap->PeekMinimum(); - if (entry == NULL) - entry = sCoreHighLoadHeap->PeekMinimum(); - } - - ASSERT(entry != NULL); - return entry->fCoreID; -} - - -static inline bool -is_task_small(Thread* thread) -{ - return thread->scheduler_data->load <= 200; -} - - -static int32 -choose_core_power_saving(Thread* thread) -{ - CoreEntry* entry; - - if (is_small_task_packing_enabled() && is_task_small(thread) - && sCoreLoadHeap->PeekMaximum() != NULL) { - // try to pack all threads on one core - if (sSmallTaskCore < 0) - sSmallTaskCore = sCoreLoadHeap->PeekMaximum()->fCoreID; - entry = &sCoreEntries[sSmallTaskCore]; - } else if (sCoreLoadHeap->PeekMinimum() != NULL) { - // run immediately on already woken core - entry = sCoreLoadHeap->PeekMinimum(); - } else if (sPackageUsageHeap->PeekMinimum() != NULL) { - // wake new core - PackageEntry* package = sPackageUsageHeap->PeekMinimum(); - entry = package->fIdleCores.Last(); - } else if (sIdlePackageList->Last() != NULL) { - // wake new package - PackageEntry* package = sIdlePackageList->Last(); - entry = package->fIdleCores.Last(); - } else { - // no idle cores, use least occupied core - entry = sCoreLoadHeap->PeekMinimum(); - if (entry == NULL) - entry = sCoreHighLoadHeap->PeekMinimum(); - } - - ASSERT(entry != NULL); - return entry->fCoreID; -} - - static inline int32 choose_core(Thread* thread) { - ASSERT(!sSingleCore); - return sChooseCore(thread); + ASSERT(!gSingleCore); + return sCurrentMode->choose_core(thread); } static inline int32 choose_cpu(int32 core) { - CPUEntry* entry = sCPUPriorityHeaps[core].PeekMinimum(); + CPUEntry* entry = gCPUPriorityHeaps[core].PeekMinimum(); ASSERT(entry != NULL); return entry->fCPUNumber; } @@ -807,10 +584,10 @@ choose_cpu(int32 core) static bool choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) { - SpinLocker coreLocker(sCoreHeapsLock); + SpinLocker coreLocker(gCoreHeapsLock); if (targetCore == -1 && targetCPU != -1) - targetCore = sCPUToCore[targetCPU]; + targetCore = gCPUToCore[targetCPU]; else if (targetCore != -1 && targetCPU == -1) targetCPU = choose_cpu(targetCore); else if (targetCore == -1 && targetCPU == -1) { @@ -818,10 +595,10 @@ choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) targetCPU = choose_cpu(targetCore); } - ASSERT(targetCore >= 0 && targetCore < sRunQueueCount); + ASSERT(targetCore >= 0 && targetCore < gRunQueueCount); ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); - int32 targetPriority = sCPUEntries[targetCPU].fPriority; + int32 targetPriority = gCPUEntries[targetCPU].fPriority; int32 threadPriority = get_effective_priority(thread); if (threadPriority > targetPriority) { @@ -836,157 +613,36 @@ choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) } -static bool -should_rebalance_low_latency(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &sCoreEntries[schedulerThreadData->previous_core]; - - // If the thread produces more than 50% of the load, leave it here. In - // such situation it is better to move other threads away. - if (schedulerThreadData->load >= coreEntry->fLoad / 2) - return false; - - // If there is high load on this core but this thread does not contribute - // significantly consider giving it to someone less busy. - if (coreEntry->fLoad > kHighLoad) { - SpinLocker coreLocker(sCoreHeapsLock); - - CoreEntry* other = sCoreLoadHeap->PeekMinimum(); - if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference) - return true; - } - - // No cpu bound threads - the situation is quite good. Make sure it - // won't get much worse... - SpinLocker coreLocker(sCoreHeapsLock); - - CoreEntry* other = sCoreLoadHeap->PeekMinimum(); - if (other == NULL) - other = sCoreHighLoadHeap->PeekMinimum(); - return coreEntry->fLoad - other->fLoad >= kLoadDifference * 2; -} - - -static bool -should_rebalance_power_saving(Thread* thread) -{ - ASSERT(!sSingleCore); - - if (thread_is_idle_thread(thread)) - return false; - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - int32 core = schedulerThreadData->previous_core; - CoreEntry* coreEntry = &sCoreEntries[core]; - - // If the thread produces more than 50% of the load, leave it here. In - // such situation it is better to move other threads away. - // Unless we are trying to pack small tasks here, in such case get rid - // of CPU hungry thread and continue packing. - if (schedulerThreadData->load >= coreEntry->fLoad / 2) - return is_small_task_packing_enabled() && sSmallTaskCore == core; - - // All cores try to give us small tasks, check whether we have enough. - if (is_small_task_packing_enabled() && sSmallTaskCore == core) { - if (coreEntry->fLoad > kHighLoad) { - if (!is_task_small(thread)) - return true; - } else if (coreEntry->fLoad > kVeryHighLoad) - disable_small_task_packing(); - } - - // Try small task packing. - if (is_small_task_packing_enabled() && is_task_small(thread)) - return sSmallTaskCore != core; - - // No cpu bound threads - the situation is quite good. Make sure it - // won't get much worse... - SpinLocker coreLocker(sCoreHeapsLock); - - CoreEntry* other = sCoreLoadHeap->PeekMinimum(); - if (other == NULL) - other = sCoreHighLoadHeap->PeekMinimum(); - return coreEntry->fLoad - other->fLoad >= kLoadDifference; -} - - static bool should_rebalance(Thread* thread) { - ASSERT(!sSingleCore); + ASSERT(!gSingleCore); if (thread_is_idle_thread(thread)) return false; - return sShouldRebalance(thread); -} - - -static void -rebalance_irqs_low_latency(void) -{ - cpu_ent* cpu = get_cpu_struct(); - SpinLocker locker(cpu->irqs_lock); - - irq_assignment* chosen = NULL; - irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); - - int32 totalLoad = 0; - while (irq != NULL) { - if (chosen == NULL || chosen->load < irq->load) - chosen = irq; - totalLoad += irq->load; - irq = (irq_assignment*)list_get_next_item(&cpu->irqs, irq); - } - - locker.Unlock(); - - if (chosen == NULL || totalLoad < kLowLoad) - return; - - SpinLocker coreLocker(sCoreHeapsLock); - CoreEntry* other = sCoreLoadHeap->PeekMinimum(); - if (other == NULL) - other = sCoreHighLoadHeap->PeekMinimum(); - coreLocker.Unlock(); - - ASSERT(other != NULL); - - int32 thisCore = sCPUToCore[smp_get_current_cpu()]; - if (other->fCoreID == thisCore) - return; - - if (other->fLoad + kLoadDifference >= sCoreEntries[thisCore].fLoad) - return; - - int32 newCPU = choose_cpu(other->fCoreID); - assign_io_interrupt_to_cpu(chosen->irq, newCPU); + return sCurrentMode->should_rebalance(thread); } static inline void compute_cpu_load(int32 cpu) { - ASSERT(!sSingleCore); + ASSERT(!gSingleCore); - int oldLoad = compute_load(sCPUEntries[cpu].fMeasureTime, - sCPUEntries[cpu].fMeasureActiveTime, sCPUEntries[cpu].fLoad); + int oldLoad = compute_load(gCPUEntries[cpu].fMeasureTime, + gCPUEntries[cpu].fMeasureActiveTime, gCPUEntries[cpu].fLoad); if (oldLoad < 0) return; - if (sCPUEntries[cpu].fLoad > kVeryHighLoad) - sRebalanceIRQs(); + if (gCPUEntries[cpu].fLoad > kVeryHighLoad) + sCurrentMode->rebalance_irqs(false); - if (oldLoad != sCPUEntries[cpu].fLoad) { - int32 core = sCPUToCore[cpu]; + if (oldLoad != gCPUEntries[cpu].fLoad) { + int32 core = gCPUToCore[cpu]; - int32 delta = sCPUEntries[cpu].fLoad - oldLoad; - atomic_add(&sCoreEntries[core].fLoad, delta); + int32 delta = gCPUEntries[cpu].fLoad - oldLoad; + atomic_add(&gCoreEntries[core].fLoad, delta); update_load_heaps(core); } @@ -1015,7 +671,7 @@ thread_goes_away(Thread* thread) schedulerThreadData->went_sleep = system_time(); schedulerThreadData->went_sleep_active - = atomic_get64(&sCoreEntries[core].fActiveTime); + = atomic_get64(&gCoreEntries[core].fActiveTime); } @@ -1031,10 +687,10 @@ should_cancel_penalty(Thread* thread) bigtime_t now = system_time(); bigtime_t wentSleep = schedulerThreadData->went_sleep; - if (wentSleep < atomic_get64(&sCoreEntries[core].fReachedIdle)) + if (wentSleep < atomic_get64(&gCoreEntries[core].fReachedIdle)) return true; - bigtime_t startedIdle = atomic_get64(&sCoreEntries[core].fStartedIdle); + bigtime_t startedIdle = atomic_get64(&gCoreEntries[core].fStartedIdle); if (startedIdle != 0) { if (wentSleep < startedIdle && now - startedIdle >= kMinimalWaitTime) return true; @@ -1046,11 +702,11 @@ should_cancel_penalty(Thread* thread) if (get_effective_priority(thread) == B_LOWEST_ACTIVE_PRIORITY) return false; - if (wentSleep < atomic_get64(&sCoreEntries[core].fReachedBottom)) + if (wentSleep < atomic_get64(&gCoreEntries[core].fReachedBottom)) return true; - bigtime_t startedBottom = atomic_get64(&sCoreEntries[core].fStartedBottom); - if (sCoreEntries[core].fStartedBottom != 0) { + bigtime_t startedBottom = atomic_get64(&gCoreEntries[core].fStartedBottom); + if (gCoreEntries[core].fStartedBottom != 0) { if (wentSleep < startedBottom && now - startedBottom >= kMinimalWaitTime) { return true; @@ -1085,7 +741,7 @@ enqueue(Thread* thread, bool newOne) int32 targetCore = -1; if (pinned) targetCPU = thread->previous_cpu->cpu_num; - else if (sSingleCore) + else if (gSingleCore) targetCore = 0; else if (schedulerThreadData->previous_core >= 0 && (!newOne || !has_cache_expired(thread)) @@ -1099,12 +755,12 @@ enqueue(Thread* thread, bool newOne) TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", thread->id, threadPriority, targetCPU, targetCore); - SpinLocker runQueueLocker(sCoreEntries[targetCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[targetCore].fLock); thread->scheduler_data->enqueued = true; if (pinned) - sPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); + gPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); else - sRunQueues[targetCore].PushBack(thread, threadPriority); + gRunQueues[targetCore].PushBack(thread, threadPriority); runQueueLocker.Unlock(); // notify listeners @@ -1150,22 +806,22 @@ put_back(Thread* thread) { compute_thread_load(thread); - int32 core = sCPUToCore[smp_get_current_cpu()]; + int32 core = gCPUToCore[smp_get_current_cpu()]; - SpinLocker runQueueLocker(sCoreEntries[core].fLock); + SpinLocker runQueueLocker(gCoreEntries[core].fLock); thread->scheduler_data->enqueued = true; if (thread->pinned_to_cpu > 0) { int32 pinnedCPU = thread->previous_cpu->cpu_num; ASSERT(pinnedCPU == smp_get_current_cpu()); - sPinnedRunQueues[pinnedCPU].PushFront(thread, + gPinnedRunQueues[pinnedCPU].PushFront(thread, get_effective_priority(thread)); } else { - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); + int32 previougCore = thread->scheduler_data->previous_core; + ASSERT(previougCore >= 0); - ASSERT(previousCore == core); - sRunQueues[previousCore].PushFront(thread, + ASSERT(previougCore == core); + gRunQueues[previougCore].PushFront(thread, get_effective_priority(thread)); } } @@ -1196,7 +852,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->priority = priority; if (thread->state == B_THREAD_RUNNING) { - SpinLocker coreLocker(sCoreHeapsLock); + SpinLocker coreLocker(gCoreHeapsLock); update_cpu_priority(thread->cpu->cpu_num, priority); } return oldPriority; @@ -1206,11 +862,11 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) // a new position. bool pinned = thread->pinned_to_cpu > 0; - int32 previousCPU = thread->previous_cpu->cpu_num; - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); + int32 previougCPU = thread->previous_cpu->cpu_num; + int32 previougCore = thread->scheduler_data->previous_core; + ASSERT(previougCore >= 0); - SpinLocker runQueueLocker(sCoreEntries[previousCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[previougCore].fLock); // the thread might have been already dequeued and is about to start // running once we release its scheduler_lock, in such case we can not @@ -1224,9 +880,9 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->scheduler_data->enqueued = false; if (pinned) - sPinnedRunQueues[previousCPU].Remove(thread); + gPinnedRunQueues[previougCPU].Remove(thread); else - sRunQueues[previousCore].Remove(thread); + gRunQueues[previougCore].Remove(thread); runQueueLocker.Unlock(); enqueue(thread, true); @@ -1338,14 +994,14 @@ compute_quantum(Thread* thread) static inline Thread* -choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) +choose_next_thread(int32 thigCPU, Thread* oldThread, bool putAtBack) { - int32 thisCore = sCPUToCore[thisCPU]; + int32 thigCore = gCPUToCore[thigCPU]; - SpinLocker runQueueLocker(sCoreEntries[thisCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[thigCore].fLock); - Thread* sharedThread = sRunQueues[thisCore].PeekMaximum(); - Thread* pinnedThread = sPinnedRunQueues[thisCPU].PeekMaximum(); + Thread* sharedThread = gRunQueues[thigCore].PeekMaximum(); + Thread* pinnedThread = gPinnedRunQueues[thigCPU].PeekMaximum(); ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); @@ -1371,34 +1027,34 @@ choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) ASSERT(sharedThread->scheduler_data->enqueued); sharedThread->scheduler_data->enqueued = false; - sRunQueues[thisCore].Remove(sharedThread); + gRunQueues[thigCore].Remove(sharedThread); return sharedThread; } ASSERT(pinnedThread->scheduler_data->enqueued); pinnedThread->scheduler_data->enqueued = false; - sPinnedRunQueues[thisCPU].Remove(pinnedThread); + gPinnedRunQueues[thigCPU].Remove(pinnedThread); return pinnedThread; } static inline void -track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) +track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thigCore) { bigtime_t now = system_time(); bigtime_t usedTime = now - oldThread->scheduler_data->quantum_start; if (thread_is_idle_thread(oldThread) && usedTime >= kMinimalWaitTime) { - atomic_set64(&sCoreEntries[thisCore].fReachedBottom, + atomic_set64(&gCoreEntries[thigCore].fReachedBottom, now - kMinimalWaitTime); - atomic_set64(&sCoreEntries[thisCore].fReachedIdle, + atomic_set64(&gCoreEntries[thigCore].fReachedIdle, now - kMinimalWaitTime); } if (get_effective_priority(oldThread) == B_LOWEST_ACTIVE_PRIORITY && usedTime >= kMinimalWaitTime) { - atomic_set64(&sCoreEntries[thisCore].fReachedBottom, + atomic_set64(&gCoreEntries[thigCore].fReachedBottom, now - kMinimalWaitTime); } @@ -1410,11 +1066,11 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) atomic_add64(&oldThread->cpu->active_time, active); oldThread->scheduler_data->measure_active_time += active; - sCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - atomic_add64(&sCoreEntries[thisCore].fActiveTime, active); + gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; + atomic_add64(&gCoreEntries[thigCore].fActiveTime, active); } - if (!sSingleCore) + if (!gSingleCore) compute_cpu_load(smp_get_current_cpu()); int32 oldPriority = get_effective_priority(oldThread); @@ -1422,16 +1078,16 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) if (thread_is_idle_thread(nextThread)) { if (!thread_is_idle_thread(oldThread)) - atomic_set64(&sCoreEntries[thisCore].fStartedIdle, now); + atomic_set64(&gCoreEntries[thigCore].fStartedIdle, now); if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - atomic_set64(&sCoreEntries[thisCore].fStartedBottom, now); + atomic_set64(&gCoreEntries[thigCore].fStartedBottom, now); } else if (nextPriority == B_LOWEST_ACTIVE_PRIORITY) { - atomic_set64(&sCoreEntries[thisCore].fStartedIdle, 0); + atomic_set64(&gCoreEntries[thigCore].fStartedIdle, 0); if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - atomic_set64(&sCoreEntries[thisCore].fStartedBottom, now); + atomic_set64(&gCoreEntries[thigCore].fStartedBottom, now); } else { - atomic_set64(&sCoreEntries[thisCore].fStartedBottom, 0); - atomic_set64(&sCoreEntries[thisCore].fStartedIdle, 0); + atomic_set64(&gCoreEntries[thigCore].fStartedBottom, 0); + atomic_set64(&gCoreEntries[thigCore].fStartedIdle, 0); } if (!thread_is_idle_thread(nextThread)) { @@ -1442,10 +1098,10 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) static inline void -update_cpu_performance(Thread* thread, int32 thisCore) +update_cpu_performance(Thread* thread, int32 thigCore) { int32 load = max_c(thread->scheduler_data->load, - sCoreEntries[thisCore].fLoad); + gCoreEntries[thigCore].fLoad); load = min_c(max_c(load, 0), kMaxLoad); if (load < kTargetLoad) { @@ -1456,7 +1112,7 @@ update_cpu_performance(Thread* thread, int32 thisCore) decrease_cpu_performance(delta); } else { - bool allowBoost = sSchedulerMode != SCHEDULER_MODE_POWER_SAVING; + bool allowBoost = !sCurrentMode->avoid_boost; allowBoost = allowBoost || thread->scheduler_data->priority_penalty > 0; int32 delta = load - kTargetLoad; @@ -1475,10 +1131,10 @@ _scheduler_reschedule(void) Thread* oldThread = thread_get_current_thread(); - int32 thisCPU = smp_get_current_cpu(); - int32 thisCore = sCPUToCore[thisCPU]; + int32 thigCPU = smp_get_current_cpu(); + int32 thigCore = gCPUToCore[thigCPU]; - TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, + TRACE("reschedule(): cpu %ld, current thread = %ld\n", thigCPU, oldThread->id); oldThread->state = oldThread->next_state; @@ -1529,7 +1185,7 @@ _scheduler_reschedule(void) // select thread with the biggest priority and enqueue back the old thread Thread* nextThread - = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, + = choose_next_thread(thigCPU, enqueueOldThread ? oldThread : NULL, putOldThreadAtBack); if (nextThread != oldThread) { if (enqueueOldThread) { @@ -1542,7 +1198,7 @@ _scheduler_reschedule(void) acquire_spinlock(&nextThread->scheduler_lock); } - TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, + TRACE("reschedule(): cpu %ld, next thread = %ld\n", thigCPU, nextThread->id); T(ScheduleThread(nextThread, oldThread)); @@ -1553,14 +1209,14 @@ _scheduler_reschedule(void) // update CPU heap { - SpinLocker coreLocker(sCoreHeapsLock); - update_cpu_priority(thisCPU, get_effective_priority(nextThread)); + SpinLocker coreLocker(gCoreHeapsLock); + update_cpu_priority(thigCPU, get_effective_priority(nextThread)); } nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; - ASSERT(nextThread->scheduler_data->previous_core == thisCore); + ASSERT(nextThread->scheduler_data->previous_core == thigCore); compute_thread_load(nextThread); @@ -1568,7 +1224,7 @@ _scheduler_reschedule(void) scheduler_update_thread_times(oldThread, nextThread); // track CPU activity - track_cpu_activity(oldThread, nextThread, thisCore); + track_cpu_activity(oldThread, nextThread, thigCore); if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; @@ -1581,10 +1237,13 @@ _scheduler_reschedule(void) add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); - update_cpu_performance(nextThread, thisCore); - } else + update_cpu_performance(nextThread, thigCore); + } else { nextThread->scheduler_data->quantum_start = system_time(); + sCurrentMode->rebalance_irqs(true); + } + modeLocker.Unlock(); if (nextThread != oldThread) scheduler_switch_thread(oldThread, nextThread); @@ -1625,13 +1284,13 @@ scheduler_on_thread_init(Thread* thread) thread->scheduler_data->Init(); if (thread_is_idle_thread(thread)) { - static int32 sIdleThreadsID; - int32 cpu = atomic_add(&sIdleThreadsID, 1); + static int32 gIdleThreadsID; + int32 cpu = atomic_add(&gIdleThreadsID, 1); thread->previous_cpu = &gCPU[cpu]; thread->pinned_to_cpu = 1; - thread->scheduler_data->previous_core = sCPUToCore[cpu]; + thread->scheduler_data->previous_core = gCPUToCore[cpu]; } } @@ -1663,33 +1322,11 @@ scheduler_set_operation_mode(scheduler_mode mode) return B_BAD_VALUE; } - const char* modeNames[] = { "low latency", "power saving" }; - dprintf("scheduler: switching to %s mode\n", modeNames[mode]); + dprintf("scheduler: switching to %s mode\n", sSchedulerModes[mode]->name); InterruptsWriteSpinLocker _(sSchedulerModeLock); - - sSchedulerMode = mode; - switch (mode) { - case SCHEDULER_MODE_LOW_LATENCY: - sDisableSmallTaskPacking = -1; - sSmallTaskCore = -1; - - sChooseCore = choose_core_low_latency; - sShouldRebalance = should_rebalance_low_latency; - sRebalanceIRQs = rebalance_irqs_low_latency; - break; - - case SCHEDULER_MODE_POWER_SAVING: - sDisableSmallTaskPacking = 0; - sSmallTaskCore = -1; - - sChooseCore = choose_core_power_saving; - sShouldRebalance = should_rebalance_power_saving; - break; - - default: - break; - } + sCurrentMode = sSchedulerModes[mode]; + sCurrentMode->switch_to_mode(); return B_OK; } @@ -1700,8 +1337,8 @@ traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) { switch (node->level) { case CPU_TOPOLOGY_SMT: - sCPUToCore[node->id] = coreID; - sCPUToPackage[node->id] = packageID; + gCPUToCore[node->id] = coreID; + gCPUToPackage[node->id] = packageID; return; case CPU_TOPOLOGY_CORE: @@ -1726,15 +1363,15 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) { cpuCount = smp_get_num_cpus(); - sCPUToCore = new(std::nothrow) int32[cpuCount]; - if (sCPUToCore == NULL) + gCPUToCore = new(std::nothrow) int32[cpuCount]; + if (gCPUToCore == NULL) return B_NO_MEMORY; - ArrayDeleter cpuToCoreDeleter(sCPUToCore); + ArrayDeleter cpuToCoreDeleter(gCPUToCore); - sCPUToPackage = new(std::nothrow) int32[cpuCount]; - if (sCPUToPackage == NULL) + gCPUToPackage = new(std::nothrow) int32[cpuCount]; + if (gCPUToPackage == NULL) return B_NO_MEMORY; - ArrayDeleter cpuToPackageDeleter(sCPUToPackage); + ArrayDeleter cpuToPackageDeleter(gCPUToPackage); coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { @@ -1768,77 +1405,77 @@ _scheduler_init() packageCount); if (result != B_OK) return result; - sRunQueueCount = coreCount; - sSingleCore = coreCount == 1; + gRunQueueCount = coreCount; + gSingleCore = coreCount == 1; // create package heap and idle package stack - sPackageEntries = new(std::nothrow) PackageEntry[packageCount]; - if (sPackageEntries == NULL) + gPackageEntries = new(std::nothrow) PackageEntry[packageCount]; + if (gPackageEntries == NULL) return B_NO_MEMORY; - ArrayDeleter packageEntriesDeleter(sPackageEntries); + ArrayDeleter packageEntriesDeleter(gPackageEntries); - sPackageUsageHeap = new(std::nothrow) PackageHeap(packageCount); - if (sPackageUsageHeap == NULL) + gPackageUsageHeap = new(std::nothrow) PackageHeap(packageCount); + if (gPackageUsageHeap == NULL) return B_NO_MEMORY; - ObjectDeleter packageHeapDeleter(sPackageUsageHeap); + ObjectDeleter packageHeapDeleter(gPackageUsageHeap); - sIdlePackageList = new(std::nothrow) IdlePackageList; - if (sIdlePackageList == NULL) + gIdlePackageList = new(std::nothrow) IdlePackageList; + if (gIdlePackageList == NULL) return B_NO_MEMORY; - ObjectDeleter packageListDeleter(sIdlePackageList); + ObjectDeleter packageListDeleter(gIdlePackageList); for (int32 i = 0; i < packageCount; i++) { - sPackageEntries[i].fPackageID = i; - sPackageEntries[i].fIdleCoreCount = coreCount / packageCount; - sPackageEntries[i].fCoreCount = coreCount / packageCount; - sIdlePackageList->Insert(&sPackageEntries[i]); + gPackageEntries[i].fPackageID = i; + gPackageEntries[i].fIdleCoreCount = coreCount / packageCount; + gPackageEntries[i].fCoreCount = coreCount / packageCount; + gIdlePackageList->Insert(&gPackageEntries[i]); } // create logical processor and core heaps - sCPUEntries = new CPUEntry[cpuCount]; - if (sCPUEntries == NULL) + gCPUEntries = new CPUEntry[cpuCount]; + if (gCPUEntries == NULL) return B_NO_MEMORY; - ArrayDeleter cpuEntriesDeleter(sCPUEntries); + ArrayDeleter cpuEntriesDeleter(gCPUEntries); - sCoreEntries = new CoreEntry[coreCount]; - if (sCoreEntries == NULL) + gCoreEntries = new CoreEntry[coreCount]; + if (gCoreEntries == NULL) return B_NO_MEMORY; - ArrayDeleter coreEntriesDeleter(sCoreEntries); + ArrayDeleter coreEntriesDeleter(gCoreEntries); - sCoreLoadHeap = new CoreLoadHeap; - if (sCoreLoadHeap == NULL) + gCoreLoadHeap = new CoreLoadHeap; + if (gCoreLoadHeap == NULL) return B_NO_MEMORY; - ObjectDeleter coreLoadHeapDeleter(sCoreLoadHeap); + ObjectDeleter coreLoadHeapDeleter(gCoreLoadHeap); - sCoreHighLoadHeap = new CoreLoadHeap(coreCount); - if (sCoreHighLoadHeap == NULL) + gCoreHighLoadHeap = new CoreLoadHeap(coreCount); + if (gCoreHighLoadHeap == NULL) return B_NO_MEMORY; - ObjectDeleter coreHighLoadHeapDeleter(sCoreHighLoadHeap); + ObjectDeleter coreHighLoadHeapDeleter(gCoreHighLoadHeap); for (int32 i = 0; i < coreCount; i++) { - sCoreEntries[i].fCoreID = i; + gCoreEntries[i].fCoreID = i; - status_t result = sCoreLoadHeap->Insert(&sCoreEntries[i], 0); + status_t result = gCoreLoadHeap->Insert(&gCoreEntries[i], 0); if (result != B_OK) return result; } - sCPUPriorityHeaps = new CPUHeap[coreCount]; - if (sCPUPriorityHeaps == NULL) + gCPUPriorityHeaps = new CPUHeap[coreCount]; + if (gCPUPriorityHeaps == NULL) return B_NO_MEMORY; - ArrayDeleter cpuPriorityHeapDeleter(sCPUPriorityHeaps); + ArrayDeleter cpuPriorityHeapDeleter(gCPUPriorityHeaps); for (int32 i = 0; i < cpuCount; i++) { - sCPUEntries[i].fCPUNumber = i; + gCPUEntries[i].fCPUNumber = i; - int32 core = sCPUToCore[i]; + int32 core = gCPUToCore[i]; - int32 package = sCPUToPackage[i]; - if (sCPUPriorityHeaps[core].PeekMaximum() == NULL) - sPackageEntries[package].fIdleCores.Insert(&sCoreEntries[core]); + int32 package = gCPUToPackage[i]; + if (gCPUPriorityHeaps[core].PeekMaximum() == NULL) + gPackageEntries[package].fIdleCores.Insert(&gCoreEntries[core]); status_t result - = sCPUPriorityHeaps[core].Insert(&sCPUEntries[i], B_IDLE_PRIORITY); + = gCPUPriorityHeaps[core].Insert(&gCPUEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; } @@ -1847,12 +1484,12 @@ _scheduler_init() TRACE("scheduler_init(): creating %" B_PRId32 " per-cpu queue%s\n", cpuCount, cpuCount != 1 ? "s" : ""); - sPinnedRunQueues = new(std::nothrow) ThreadRunQueue[cpuCount]; - if (sPinnedRunQueues == NULL) + gPinnedRunQueues = new(std::nothrow) ThreadRunQueue[cpuCount]; + if (gPinnedRunQueues == NULL) return B_NO_MEMORY; - ArrayDeleter pinnedRunQueuesDeleter(sPinnedRunQueues); + ArrayDeleter pinnedRunQueuesDeleter(gPinnedRunQueues); for (int i = 0; i < cpuCount; i++) { - status_t result = sPinnedRunQueues[i].GetInitStatus(); + status_t result = gPinnedRunQueues[i].GetInitStatus(); if (result != B_OK) return result; } @@ -1861,12 +1498,12 @@ _scheduler_init() TRACE("scheduler_init(): creating %" B_PRId32 " per-core queue%s\n", coreCount, coreCount != 1 ? "s" : ""); - sRunQueues = new(std::nothrow) ThreadRunQueue[coreCount]; - if (sRunQueues == NULL) + gRunQueues = new(std::nothrow) ThreadRunQueue[coreCount]; + if (gRunQueues == NULL) return B_NO_MEMORY; - ArrayDeleter runQueuesDeleter(sRunQueues); + ArrayDeleter runQueuesDeleter(gRunQueues); for (int i = 0; i < coreCount; i++) { - status_t result = sRunQueues[i].GetInitStatus(); + status_t result = gRunQueues[i].GetInitStatus(); if (result != B_OK) return result; } @@ -1882,7 +1519,7 @@ _scheduler_init() add_debugger_command_etc("cpu_heap", &dump_cpu_heap, "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", 0); - if (!sSingleCore) { + if (!gSingleCore) { add_debugger_command_etc("idle_cores", &dump_idle_cores, "List idle cores", "\nList idle cores", 0); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index a105eb8a52..2eaead2e79 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2011, Ingo Weinhold, ingo_weinhold@gmx.de. * Distributed under the terms of the MIT License. */ @@ -6,9 +7,167 @@ #define KERNEL_SCHEDULER_COMMON_H +#include #include +#include #include +#include #include +#include + +#include "RunQueue.h" + + +#define CACHE_LINE_ALIGN __attribute__((aligned(64))) + + +//#define TRACE_SCHEDULER +#ifdef TRACE_SCHEDULER +# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) +#else +# define TRACE(...) do { } while (false) +#endif + + +namespace Scheduler { + + +const bigtime_t kThreadQuantum = 1000; +const bigtime_t kMinThreadQuantum = 3000; +const bigtime_t kMaxThreadQuantum = 10000; + +const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; + +const bigtime_t kCacheExpire = 100000; + +const int kTargetLoad = kMaxLoad * 55 / 100; +const int kHighLoad = kMaxLoad * 70 / 100; +const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; +const int kLoadDifference = kMaxLoad * 20 / 100; +const int kLowLoad = kLoadDifference / 2; + +extern bool gSingleCore; + +// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical +// processors in the heap belong to. Since there are no cache affinity issues +// at this level and the run queue is shared among all logical processors on +// the core the only real concern is to make lower priority threads give way to +// the higher priority threads. +struct CPUEntry : public MinMaxHeapLinkImpl { + CPUEntry(); + + int32 fCPUNumber; + + int32 fPriority; + + bigtime_t fMeasureActiveTime; + bigtime_t fMeasureTime; + + int32 fLoad; +} CACHE_LINE_ALIGN; +typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; + +extern CPUEntry* gCPUEntries; +extern CPUHeap* gCPUPriorityHeaps; + +struct CoreEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { + CoreEntry(); + + int32 fCoreID; + + spinlock fLock; + + bigtime_t fStartedBottom; + bigtime_t fReachedBottom; + bigtime_t fStartedIdle; + bigtime_t fReachedIdle; + + bigtime_t fActiveTime; + + int32 fLoad; +} CACHE_LINE_ALIGN; +typedef MinMaxHeap CoreLoadHeap; + +extern CoreEntry* gCoreEntries; +extern CoreLoadHeap* gCoreLoadHeap; +extern CoreLoadHeap* gCoreHighLoadHeap; +extern spinlock gCoreHeapsLock; + +// sPackageUsageHeap is used to decide which core should be woken up from the +// idle state. When aiming for performance we should use as many packages as +// possible with as little cores active in each package as possible (so that the +// package can enter any boost mode if it has one and the active core have more +// of the shared cache for themselves. If power saving is the main priority we +// should keep active cores on as little packages as possible (so that other +// packages can go to the deep state of sleep). The heap stores only packages +// with at least one core active and one core idle. The packages with all cores +// idle are stored in sPackageIdleList (in LIFO manner). +struct PackageEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { + PackageEntry(); + + int32 fPackageID; + + DoublyLinkedList fIdleCores; + int32 fIdleCoreCount; + + int32 fCoreCount; +} CACHE_LINE_ALIGN; +typedef MinMaxHeap PackageHeap; +typedef DoublyLinkedList IdlePackageList; + +extern PackageEntry* gPackageEntries; +extern PackageHeap* gPackageUsageHeap; +extern IdlePackageList* gIdlePackageList; +extern spinlock gIdlePackageLock; + +// The run queues. Holds the threads ready to run ordered by priority. +// One queue per schedulable target per core. Additionally, each +// logical processor has its sPinnedRunQueues used for scheduling +// pinned threads. +typedef RunQueue CACHE_LINE_ALIGN + ThreadRunQueue; + +extern ThreadRunQueue* gRunQueues; +extern ThreadRunQueue* gPinnedRunQueues; +extern int32 gRunQueueCount; + +// Since CPU IDs used internally by the kernel bear no relation to the actual +// CPU topology the following arrays are used to efficiently get the core +// and the package that CPU in question belongs to. +extern int32* gCPUToCore; +extern int32* gCPUToPackage; + + +} // namespace Scheduler + + +struct scheduler_thread_data { + inline scheduler_thread_data(); + void Init(); + + int32 priority_penalty; + int32 additional_penalty; + + bool lost_cpu; + bool cpu_bound; + + bigtime_t time_left; + bigtime_t stolen_time; + bigtime_t quantum_start; + + bigtime_t measure_active_time; + bigtime_t measure_time; + int32 load; + + bigtime_t went_sleep; + bigtime_t went_sleep_active; + + int32 previous_core; + + bool enqueued; +}; /*! Switches the currently running thread. diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h new file mode 100644 index 0000000000..3b13a9c371 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -0,0 +1,29 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_MODES_H +#define KERNEL_SCHEDULER_MODES_H + + +#include +#include + + +struct scheduler_mode_operations { + const char* name; + + bool avoid_boost; + + void (*switch_to_mode)(void); + bool (*has_cache_expired)(Thread* thread); + int32 (*choose_core)(Thread* thread); + bool (*should_rebalance)(Thread* thread); + void (*rebalance_irqs)(bool idle); +}; + +extern struct scheduler_mode_operations gSchedulerLowLatencyMode; +extern struct scheduler_mode_operations gSchedulerPowerSavingMode; + +#endif // KERNEL_SCHEDULER_MODES_H + From c4ac37a35e449a07393867fed7278f377cca8dbf Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 10:28:59 +0100 Subject: [PATCH 124/273] scheduler: Try to pack IRQs in power saving mode --- src/system/kernel/scheduler/low_latency.cpp | 11 +-- src/system/kernel/scheduler/power_saving.cpp | 74 +++++++++++++++++++- src/system/kernel/scheduler/scheduler.cpp | 59 ++++++++-------- 3 files changed, 107 insertions(+), 37 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 7a0ee48570..bc14cadb5c 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -124,19 +124,20 @@ rebalance_irqs(bool idle) CoreEntry* other = gCoreLoadHeap->PeekMinimum(); if (other == NULL) other = gCoreHighLoadHeap->PeekMinimum(); + + int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; coreLocker.Unlock(); ASSERT(other != NULL); - int32 thigCore = gCPUToCore[smp_get_current_cpu()]; - if (other->fCoreID == thigCore) + int32 thisCore = gCPUToCore[smp_get_current_cpu()]; + if (other->fCoreID == thisCore) return; - if (other->fLoad + kLoadDifference >= gCoreEntries[thigCore].fLoad) + if (other->fLoad + kLoadDifference >= gCoreEntries[thisCore].fLoad) return; - coreLocker.Lock(); - gCPUPriorityHeaps[other->fCoreID].PeekMinimum(); + assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index aed4e5fd23..0f6b06f3b3 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -28,7 +28,6 @@ has_cache_expired(Thread* thread) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; ASSERT(schedulerThreadData->previous_core >= 0); - CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; return system_time() - schedulerThreadData->went_sleep > kCacheExpire; } @@ -149,6 +148,76 @@ should_rebalance(Thread* thread) } +static inline void +pack_irqs(void) +{ + cpu_ent* cpu = get_cpu_struct(); + int32 core = gCPUToCore[cpu->cpu_num]; + + SpinLocker locker(cpu->irqs_lock); + while (sSmallTaskCore != core && list_get_first_item(&cpu->irqs) != NULL) { + irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); + locker.Unlock(); + + SpinLocker coreLocker(gCoreHeapsLock); + int32 newCPU + = gCPUPriorityHeaps[sSmallTaskCore].PeekMinimum()->fCPUNumber; + coreLocker.Unlock(); + + if (newCPU != cpu->cpu_num) + assign_io_interrupt_to_cpu(irq->irq, newCPU); + + locker.Lock(); + } +} + + +static void +rebalance_irqs(bool idle) +{ + if (idle && !is_small_task_packing_enabled() && sSmallTaskCore != -1) { + pack_irqs(); + return; + } + + if (idle) + return; + + cpu_ent* cpu = get_cpu_struct(); + SpinLocker locker(cpu->irqs_lock); + + irq_assignment* chosen = NULL; + irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); + + while (irq != NULL) { + if (chosen == NULL || chosen->load < irq->load) + chosen = irq; + irq = (irq_assignment*)list_get_next_item(&cpu->irqs, irq); + } + + locker.Unlock(); + + if (chosen == NULL || chosen->load < kLowLoad) + return; + + SpinLocker coreLocker(gCoreHeapsLock); + CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + if (other == NULL) + return; + int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; + coreLocker.Unlock(); + + int32 thisCore = gCPUToCore[smp_get_current_cpu()]; + if (other->fCoreID == thisCore) + return; + + if (other->fLoad + kLoadDifference >= gCoreEntries[thisCore].fLoad) + return; + + assign_io_interrupt_to_cpu(chosen->irq, newCPU); +} + + scheduler_mode_operations gSchedulerPowerSavingMode = { "power saving", @@ -158,7 +227,6 @@ scheduler_mode_operations gSchedulerPowerSavingMode = { has_cache_expired, choose_core, should_rebalance, - NULL, + rebalance_irqs, }; - diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 394706ef47..c478870b83 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -994,14 +994,14 @@ compute_quantum(Thread* thread) static inline Thread* -choose_next_thread(int32 thigCPU, Thread* oldThread, bool putAtBack) +choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) { - int32 thigCore = gCPUToCore[thigCPU]; + int32 thisCore = gCPUToCore[thisCPU]; - SpinLocker runQueueLocker(gCoreEntries[thigCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[thisCore].fLock); - Thread* sharedThread = gRunQueues[thigCore].PeekMaximum(); - Thread* pinnedThread = gPinnedRunQueues[thigCPU].PeekMaximum(); + Thread* sharedThread = gRunQueues[thisCore].PeekMaximum(); + Thread* pinnedThread = gPinnedRunQueues[thisCPU].PeekMaximum(); ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); @@ -1027,34 +1027,34 @@ choose_next_thread(int32 thigCPU, Thread* oldThread, bool putAtBack) ASSERT(sharedThread->scheduler_data->enqueued); sharedThread->scheduler_data->enqueued = false; - gRunQueues[thigCore].Remove(sharedThread); + gRunQueues[thisCore].Remove(sharedThread); return sharedThread; } ASSERT(pinnedThread->scheduler_data->enqueued); pinnedThread->scheduler_data->enqueued = false; - gPinnedRunQueues[thigCPU].Remove(pinnedThread); + gPinnedRunQueues[thisCPU].Remove(pinnedThread); return pinnedThread; } static inline void -track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thigCore) +track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) { bigtime_t now = system_time(); bigtime_t usedTime = now - oldThread->scheduler_data->quantum_start; if (thread_is_idle_thread(oldThread) && usedTime >= kMinimalWaitTime) { - atomic_set64(&gCoreEntries[thigCore].fReachedBottom, + atomic_set64(&gCoreEntries[thisCore].fReachedBottom, now - kMinimalWaitTime); - atomic_set64(&gCoreEntries[thigCore].fReachedIdle, + atomic_set64(&gCoreEntries[thisCore].fReachedIdle, now - kMinimalWaitTime); } if (get_effective_priority(oldThread) == B_LOWEST_ACTIVE_PRIORITY && usedTime >= kMinimalWaitTime) { - atomic_set64(&gCoreEntries[thigCore].fReachedBottom, + atomic_set64(&gCoreEntries[thisCore].fReachedBottom, now - kMinimalWaitTime); } @@ -1067,7 +1067,7 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thigCore) oldThread->scheduler_data->measure_active_time += active; gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - atomic_add64(&gCoreEntries[thigCore].fActiveTime, active); + atomic_add64(&gCoreEntries[thisCore].fActiveTime, active); } if (!gSingleCore) @@ -1078,16 +1078,16 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thigCore) if (thread_is_idle_thread(nextThread)) { if (!thread_is_idle_thread(oldThread)) - atomic_set64(&gCoreEntries[thigCore].fStartedIdle, now); + atomic_set64(&gCoreEntries[thisCore].fStartedIdle, now); if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - atomic_set64(&gCoreEntries[thigCore].fStartedBottom, now); + atomic_set64(&gCoreEntries[thisCore].fStartedBottom, now); } else if (nextPriority == B_LOWEST_ACTIVE_PRIORITY) { - atomic_set64(&gCoreEntries[thigCore].fStartedIdle, 0); + atomic_set64(&gCoreEntries[thisCore].fStartedIdle, 0); if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - atomic_set64(&gCoreEntries[thigCore].fStartedBottom, now); + atomic_set64(&gCoreEntries[thisCore].fStartedBottom, now); } else { - atomic_set64(&gCoreEntries[thigCore].fStartedBottom, 0); - atomic_set64(&gCoreEntries[thigCore].fStartedIdle, 0); + atomic_set64(&gCoreEntries[thisCore].fStartedBottom, 0); + atomic_set64(&gCoreEntries[thisCore].fStartedIdle, 0); } if (!thread_is_idle_thread(nextThread)) { @@ -1098,10 +1098,10 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thigCore) static inline void -update_cpu_performance(Thread* thread, int32 thigCore) +update_cpu_performance(Thread* thread, int32 thisCore) { int32 load = max_c(thread->scheduler_data->load, - gCoreEntries[thigCore].fLoad); + gCoreEntries[thisCore].fLoad); load = min_c(max_c(load, 0), kMaxLoad); if (load < kTargetLoad) { @@ -1131,10 +1131,10 @@ _scheduler_reschedule(void) Thread* oldThread = thread_get_current_thread(); - int32 thigCPU = smp_get_current_cpu(); - int32 thigCore = gCPUToCore[thigCPU]; + int32 thisCPU = smp_get_current_cpu(); + int32 thisCore = gCPUToCore[thisCPU]; - TRACE("reschedule(): cpu %ld, current thread = %ld\n", thigCPU, + TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, oldThread->id); oldThread->state = oldThread->next_state; @@ -1185,7 +1185,7 @@ _scheduler_reschedule(void) // select thread with the biggest priority and enqueue back the old thread Thread* nextThread - = choose_next_thread(thigCPU, enqueueOldThread ? oldThread : NULL, + = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, putOldThreadAtBack); if (nextThread != oldThread) { if (enqueueOldThread) { @@ -1198,7 +1198,7 @@ _scheduler_reschedule(void) acquire_spinlock(&nextThread->scheduler_lock); } - TRACE("reschedule(): cpu %ld, next thread = %ld\n", thigCPU, + TRACE("reschedule(): cpu %ld, next thread = %ld\n", thisCPU, nextThread->id); T(ScheduleThread(nextThread, oldThread)); @@ -1210,13 +1210,13 @@ _scheduler_reschedule(void) // update CPU heap { SpinLocker coreLocker(gCoreHeapsLock); - update_cpu_priority(thigCPU, get_effective_priority(nextThread)); + update_cpu_priority(thisCPU, get_effective_priority(nextThread)); } nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; - ASSERT(nextThread->scheduler_data->previous_core == thigCore); + ASSERT(nextThread->scheduler_data->previous_core == thisCore); compute_thread_load(nextThread); @@ -1224,7 +1224,7 @@ _scheduler_reschedule(void) scheduler_update_thread_times(oldThread, nextThread); // track CPU activity - track_cpu_activity(oldThread, nextThread, thigCore); + track_cpu_activity(oldThread, nextThread, thisCore); if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; @@ -1237,7 +1237,7 @@ _scheduler_reschedule(void) add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); - update_cpu_performance(nextThread, thigCore); + update_cpu_performance(nextThread, thisCore); } else { nextThread->scheduler_data->quantum_start = system_time(); @@ -1631,3 +1631,4 @@ _user_estimate_max_scheduling_latency(thread_id id) return 2 * kMinThreadQuantum; } + From 57d5d678f275094da95f9725e588a279224ff4d0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 15:11:00 +0100 Subject: [PATCH 125/273] x86_64: Fix atomic functions --- src/system/libroot/os/arch/x86_64/atomic.S | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/system/libroot/os/arch/x86_64/atomic.S b/src/system/libroot/os/arch/x86_64/atomic.S index 5c905ceb01..b88af32987 100644 --- a/src/system/libroot/os/arch/x86_64/atomic.S +++ b/src/system/libroot/os/arch/x86_64/atomic.S @@ -21,7 +21,8 @@ FUNCTION_END(atomic_set) /* int32 atomic_get_and_set(int32* value, int32 newValue) */ FUNCTION(atomic_get_and_set): - xchgl %esi, (%rdi) + movl %esi, %eax + xchgl %eax, (%rdi) ret FUNCTION_END(atomic_get_and_set) @@ -37,8 +38,9 @@ FUNCTION_END(atomic_test_and_set) /* int32 atomic_add(int32* value, int32 addValue) */ FUNCTION(atomic_add): + movl %esi, %eax lock - xaddl %esi, (%rdi) + xaddl %eax, (%rdi) ret FUNCTION_END(atomic_add) @@ -75,6 +77,7 @@ FUNCTION_END(atomic_or) FUNCTION(atomic_get): movl (%rdi), %eax lfence + ret FUNCTION_END(atomic_get) @@ -88,7 +91,8 @@ FUNCTION_END(atomic_set64) /* int64 atomic_get_and_set64(int64* value, int64 newValue) */ FUNCTION(atomic_get_and_set64): - xchgq %rsi, (%rdi) + movq %rsi, %rax + xchgq %rax, (%rdi) ret FUNCTION_END(atomic_get_and_set64) @@ -144,5 +148,6 @@ FUNCTION_END(atomic_or64) FUNCTION(atomic_get64): movq (%rdi), %rax lfence + ret FUNCTION_END(atomic_get64) From 3eb4224bf6d658b0e95412d348e6656fefe92f65 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 17:53:39 +0100 Subject: [PATCH 126/273] kernel: Make sure mutex::holder is set to a valid value --- src/system/kernel/locks/lock.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/locks/lock.cpp b/src/system/kernel/locks/lock.cpp index 6b3ac5ff70..3240549fbd 100644 --- a/src/system/kernel/locks/lock.cpp +++ b/src/system/kernel/locks/lock.cpp @@ -775,6 +775,9 @@ _mutex_unlock(mutex* lock) lock->waiters = waiter->next; if (lock->waiters != NULL) lock->waiters->last = waiter->last; +#if KDEBUG + thread_id unblockedThread = waiter->thread->id; +#endif // unblock thread thread_unblock(waiter->thread, B_OK); @@ -784,7 +787,7 @@ _mutex_unlock(mutex* lock) // actually reflects the current situation, setting it to -1 would // cause a race condition, since another locker could think the lock // is not held by anyone. - lock->holder = waiter->thread->id; + lock->holder = unblockedThread; #endif } else { // We've acquired the spinlock before the locker that is going to wait. From ecfd44493554c78001bbe001d35637b7eab0ffb6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 20:32:53 +0100 Subject: [PATCH 127/273] scheduler: Improve power saving mode * Remove possibility to temporarily disable small task packing. * When small task packing target gets overloaded continue packing threads on another core, but avoid migrating the already packed ones. Scheduler still tends to needlessly migrate threads to another cores when under heavier load, but it is now much better than before. --- src/system/kernel/scheduler/power_saving.cpp | 110 ++++++++---------- src/system/kernel/scheduler/scheduler.cpp | 2 +- .../kernel/scheduler/scheduler_common.h | 3 +- 3 files changed, 51 insertions(+), 64 deletions(-) diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 0f6b06f3b3..a3fba22f1f 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -13,7 +13,6 @@ using namespace Scheduler; -static bigtime_t sDisableSmallTaskPacking; static int32 sSmallTaskCore; @@ -32,54 +31,47 @@ has_cache_expired(Thread* thread) } -static inline bool -is_small_task_packing_enabled(void) -{ - if (sDisableSmallTaskPacking == -1) - return false; - return sDisableSmallTaskPacking < system_time(); -} - - -static inline void -disable_small_task_packing(void) -{ - ASSERT(!gSingleCore); - - ASSERT(is_small_task_packing_enabled()); - ASSERT(sSmallTaskCore == gCPUToCore[smp_get_current_cpu()]); - - sDisableSmallTaskPacking = system_time() + kThreadQuantum * 100; - sSmallTaskCore = -1; -} - - -static inline bool -is_task_small(Thread* thread) -{ - return thread->scheduler_data->load <= 200; -} - - static void switch_to_mode(void) { - sDisableSmallTaskPacking = -1; sSmallTaskCore = -1; } +static bool +try_small_task_packing(Thread* thread) +{ + int32 core = sSmallTaskCore; + return (core == -1 && gCoreLoadHeap->PeekMaximum() != NULL) + || (core != -1 + && gCoreEntries[core].fLoad + thread->scheduler_data->load + < kHighLoad); +} + + +static int32 +choose_small_task_core(void) +{ + CoreEntry* candidate = gCoreLoadHeap->PeekMaximum(); + if (candidate == NULL) + return sSmallTaskCore; + + int32 core = candidate->fCoreID; + int32 smallTaskCore = atomic_test_and_set(&sSmallTaskCore, core, -1); + if (smallTaskCore == -1) + return core; + return smallTaskCore; +} + + static int32 choose_core(Thread* thread) { CoreEntry* entry; - if (is_small_task_packing_enabled() && is_task_small(thread) - && gCoreLoadHeap->PeekMaximum() != NULL) { + if (try_small_task_packing(thread)) { // try to pack all threads on one core - if (sSmallTaskCore < 0) - sSmallTaskCore = gCoreLoadHeap->PeekMaximum()->fCoreID; - entry = &gCoreEntries[sSmallTaskCore]; + entry = &gCoreEntries[choose_small_task_core()]; } else if (gCoreLoadHeap->PeekMinimum() != NULL) { // run immediately on already woken core entry = gCoreLoadHeap->PeekMinimum(); @@ -117,34 +109,28 @@ should_rebalance(Thread* thread) int32 core = schedulerThreadData->previous_core; CoreEntry* coreEntry = &gCoreEntries[core]; - // If the thread produces more than 50% of the load, leave it here. In - // such situation it is better to move other threads away. - // Unless we are trying to pack small tasks here, in such case get rid - // of CPU hungry thread and continue packing. - if (schedulerThreadData->load >= coreEntry->fLoad / 2) - return is_small_task_packing_enabled() && sSmallTaskCore == core; - - // All cores try to give us small tasks, check whether we have enough. - if (is_small_task_packing_enabled() && sSmallTaskCore == core) { - if (coreEntry->fLoad > kHighLoad) { - if (!is_task_small(thread)) + if (coreEntry->fLoad > kHighLoad) { + if (sSmallTaskCore == core) { + SpinLocker coreLocker(gCoreHeapsLock); + CoreEntry* other = gCoreLoadHeap->PeekMaximum(); + + if (other == NULL) + sSmallTaskCore = -1; + else if (coreEntry->fLoad - schedulerThreadData->load < kHighLoad) return true; - } else if (coreEntry->fLoad > kVeryHighLoad) - disable_small_task_packing(); + else + sSmallTaskCore = other->fCoreID; + return coreEntry->fLoad > kVeryHighLoad; + } + } else if (coreEntry->fLoad < kHighLoad) { + int32 newCore = choose_small_task_core(); + return newCore != core; } - // Try small task packing. - if (is_small_task_packing_enabled() && is_task_small(thread)) - return sSmallTaskCore != core; - - // No cpu bound threads - the situation is quite good. Make sure it - // won't get much worse... - SpinLocker coreLocker(gCoreHeapsLock); - - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreHighLoadHeap->PeekMinimum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); - return coreEntry->fLoad - other->fLoad >= kLoadDifference; + return false; + return coreEntry->fLoad - other->fLoad >= kLoadDifference / 2; } @@ -175,12 +161,12 @@ pack_irqs(void) static void rebalance_irqs(bool idle) { - if (idle && !is_small_task_packing_enabled() && sSmallTaskCore != -1) { + if (idle && sSmallTaskCore != -1) { pack_irqs(); return; } - if (idle) + if (idle || sSmallTaskCore != -1) return; cpu_ent* cpu = get_cpu_struct(); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index c478870b83..98c87bf5c6 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1508,7 +1508,7 @@ _scheduler_init() return result; } -#if 1 +#if 0 scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); #else scheduler_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 2eaead2e79..7d23364730 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -40,11 +40,12 @@ const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; const bigtime_t kCacheExpire = 100000; +const int kLowLoad = kMaxLoad * 20 / 100; const int kTargetLoad = kMaxLoad * 55 / 100; const int kHighLoad = kMaxLoad * 70 / 100; const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; + const int kLoadDifference = kMaxLoad * 20 / 100; -const int kLowLoad = kLoadDifference / 2; extern bool gSingleCore; From 683b9bbf07cd85105ca1ab9e926f140d6e78b7fc Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 21:21:31 +0100 Subject: [PATCH 128/273] scheduler: Improve power saving mode, part 2 Consequences of committing & pushing too quickly... --- src/system/kernel/scheduler/power_saving.cpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index a3fba22f1f..ac848e9ab0 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -110,8 +110,8 @@ should_rebalance(Thread* thread) CoreEntry* coreEntry = &gCoreEntries[core]; if (coreEntry->fLoad > kHighLoad) { + SpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { - SpinLocker coreLocker(gCoreHeapsLock); CoreEntry* other = gCoreLoadHeap->PeekMaximum(); if (other == NULL) @@ -122,15 +122,14 @@ should_rebalance(Thread* thread) sSmallTaskCore = other->fCoreID; return coreEntry->fLoad > kVeryHighLoad; } - } else if (coreEntry->fLoad < kHighLoad) { - int32 newCore = choose_small_task_core(); - return newCore != core; + + CoreEntry* other = gCoreHighLoadHeap->PeekMinimum(); + if (other == NULL) + other = gCoreHighLoadHeap->PeekMaximum(); + return coreEntry->fLoad - other->fLoad >= kLoadDifference / 2; } - CoreEntry* other = gCoreHighLoadHeap->PeekMinimum(); - if (other == NULL) - return false; - return coreEntry->fLoad - other->fLoad >= kLoadDifference / 2; + return choose_small_task_core() != core; } From cd1e7814a92caa9060f161e2c910f9d42b92ab9e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 21:49:31 +0100 Subject: [PATCH 129/273] scheduler: Reduce scheduler mode lock cache bouncing --- src/system/kernel/scheduler/scheduler.cpp | 41 ++++++++++++++++--- .../kernel/scheduler/scheduler_common.h | 2 + 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 98c87bf5c6..aacd89d35d 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -43,7 +43,6 @@ static bool sSchedulerEnabled; SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -static rw_spinlock sSchedulerModeLock = B_RW_SPINLOCK_INITIALIZER; static struct scheduler_mode_operations* sCurrentMode; static struct scheduler_mode_operations* sSchedulerModes[] = { &gSchedulerLowLatencyMode, @@ -74,6 +73,29 @@ int32 gRunQueueCount; int32* gCPUToCore; int32* gCPUToPackage; +class SchedulerModeLocker : public ReadSpinLocker { +public: + inline SchedulerModeLocker(bool alreadyLocked = false, + bool lockIfNotLocked = true) + : + ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, + alreadyLocked, lockIfNotLocked) + { + } +}; + +class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { +public: + inline InterruptsSchedulerModeLocker(bool alreadyLocked = false, + bool lockIfNotLocked = true) + : + InterruptsReadSpinLocker( + gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, + alreadyLocked, lockIfNotLocked) + { + } +}; + } // namespace Scheduler @@ -84,6 +106,7 @@ CPUEntry::CPUEntry() fMeasureTime(0), fLoad(0) { + B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); } @@ -784,7 +807,7 @@ enqueue(Thread* thread, bool newOne) void scheduler_enqueue_in_run_queue(Thread *thread) { - InterruptsReadSpinLocker modeLocker(sSchedulerModeLock); + InterruptsSchedulerModeLocker _; TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); @@ -833,7 +856,7 @@ int32 scheduler_set_thread_priority(Thread *thread, int32 priority) { InterruptsSpinLocker _(thread->scheduler_lock); - InterruptsReadSpinLocker modeLocker(sSchedulerModeLock); + SchedulerModeLocker modeLocker; int32 oldPriority = thread->priority; @@ -1127,7 +1150,9 @@ update_cpu_performance(Thread* thread, int32 thisCore) static void _scheduler_reschedule(void) { - InterruptsReadSpinLocker modeLocker(sSchedulerModeLock); + ASSERT(!are_interrupts_enabled()); + + SchedulerModeLocker modeLocker; Thread* oldThread = thread_get_current_thread(); @@ -1324,10 +1349,16 @@ scheduler_set_operation_mode(scheduler_mode mode) dprintf("scheduler: switching to %s mode\n", sSchedulerModes[mode]->name); - InterruptsWriteSpinLocker _(sSchedulerModeLock); + InterruptsLocker _; + for (int32_t i = 0; i < smp_get_num_cpus(); i++) + acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + sCurrentMode = sSchedulerModes[mode]; sCurrentMode->switch_to_mode(); + for (int32_t i = 0; i < smp_get_num_cpus(); i++) + release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 7d23364730..ea3aa422ad 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -65,6 +65,8 @@ struct CPUEntry : public MinMaxHeapLinkImpl { bigtime_t fMeasureTime; int32 fLoad; + + rw_spinlock fSchedulerModeLock; } CACHE_LINE_ALIGN; typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; From 308f594e2a5d3674bd019a7620e193ab0a845e6f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 20 Nov 2013 23:32:40 +0100 Subject: [PATCH 130/273] kernel, libroot: Make scheduler modes interface public --- headers/os/kernel/scheduler.h | 11 +++++++++ headers/private/kernel/kscheduler.h | 9 +++---- headers/private/system/syscalls.h | 3 +++ .../processcontroller/ProcessController.cpp | 24 +++++++++++++++++++ src/system/kernel/scheduler/scheduler.cpp | 23 +++++++++++++----- src/system/libroot/os/scheduler.c | 14 +++++++++++ 6 files changed, 74 insertions(+), 10 deletions(-) diff --git a/headers/os/kernel/scheduler.h b/headers/os/kernel/scheduler.h index f55eb2ff57..be3508775e 100644 --- a/headers/os/kernel/scheduler.h +++ b/headers/os/kernel/scheduler.h @@ -49,6 +49,11 @@ enum be_task_flags { B_MIDI_PROCESSING = 0x800 }; +enum scheduler_mode { + SCHEDULER_MODE_LOW_LATENCY, + SCHEDULER_MODE_POWER_SAVING, +}; + #if defined(__cplusplus) extern "C" { @@ -58,6 +63,9 @@ int32 suggest_thread_priority(uint32 task_flags = B_DEFAULT_MEDIA_PRIORITY, bigtime_t estimate_max_scheduling_latency(thread_id th = -1); /* default is current thread */ +status_t set_scheduler_mode(int32 mode); +int32 get_scheduler_mode(void); + } #else @@ -67,6 +75,9 @@ int32 suggest_thread_priority(uint32 what, int32 period, bigtime_t jitter, bigtime_t estimate_max_scheduling_latency(thread_id th); /* default is current thread */ +status_t set_scheduler_mode(int32 mode); +int32 get_scheduler_mode(void); + #endif #endif // SCHEDULER_H diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index dc2bb88093..92a82e9ccc 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -18,12 +18,10 @@ struct scheduling_analysis; struct SchedulerListener; -typedef enum scheduler_mode { +enum scheduler_mode { SCHEDULER_MODE_LOW_LATENCY, SCHEDULER_MODE_POWER_SAVING, - // ... - SCHEDULER_MODE_COUNT -} scheduler_mode; +}; #ifdef __cplusplus @@ -95,6 +93,9 @@ bigtime_t _user_estimate_max_scheduling_latency(thread_id thread); status_t _user_analyze_scheduling(bigtime_t from, bigtime_t until, void* buffer, size_t size, struct scheduling_analysis* analysis); +status_t _user_set_scheduler_mode(int32 mode); +int32 _user_get_scheduler_mode(void); + #ifdef __cplusplus } #endif diff --git a/headers/private/system/syscalls.h b/headers/private/system/syscalls.h index 438a3ea310..6b1abb6b0b 100644 --- a/headers/private/system/syscalls.h +++ b/headers/private/system/syscalls.h @@ -192,6 +192,9 @@ extern status_t _kern_unblock_threads(thread_id* threads, uint32 count, extern bigtime_t _kern_estimate_max_scheduling_latency(thread_id thread); +extern status_t _kern_set_scheduler_mode(int32 mode); +extern int32 _kern_get_scheduler_mode(void); + // user/group functions extern gid_t _kern_getgid(bool effective); extern uid_t _kern_getuid(bool effective); diff --git a/src/apps/processcontroller/ProcessController.cpp b/src/apps/processcontroller/ProcessController.cpp index 4925f065f3..3987582371 100644 --- a/src/apps/processcontroller/ProcessController.cpp +++ b/src/apps/processcontroller/ProcessController.cpp @@ -42,6 +42,7 @@ #include #include +#include #include #include "AutoIcon.h" @@ -441,6 +442,14 @@ ProcessController::MessageReceived(BMessage *message) break; } + case 'Schd': + { + int32 mode; + if (message->FindInt32 ("mode", &mode) == B_OK) + set_scheduler_mode(mode); + break; + } + case B_ABOUT_REQUESTED: AboutRequested(); break; @@ -786,6 +795,21 @@ thread_popup(void *arg) addtopbottom (new BSeparatorItem ()); } + // Scheduler modes + static const char* schedulerModes[] = { "Low Latency", "Power Saving" }; + unsigned int modesCount = sizeof(schedulerModes) / sizeof(const char*); + int32 currentMode = get_scheduler_mode(); + for (unsigned int i = 0; i < modesCount; i++) { + BMessage* m = new BMessage('Schd'); + m->AddInt32("mode", i); + item = new BMenuItem(B_TRANSLATE(schedulerModes[i]), m); + if ((uint32)currentMode == i) + item->SetMarked(true); + item->SetTarget(gPCView); + addtopbottom(item); + } + addtopbottom(new BSeparatorItem()); + if (!be_roster->IsRunning(kTrackerSig)) { item = new IconMenuItem(gPCView->fTrackerIcon, B_TRANSLATE("Restart Tracker"), new BMessage('Trac')); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index aacd89d35d..b8f738d695 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -43,8 +43,9 @@ static bool sSchedulerEnabled; SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -static struct scheduler_mode_operations* sCurrentMode; -static struct scheduler_mode_operations* sSchedulerModes[] = { +static scheduler_mode sCurrentModeID; +static scheduler_mode_operations* sCurrentMode; +static scheduler_mode_operations* sSchedulerModes[] = { &gSchedulerLowLatencyMode, &gSchedulerPowerSavingMode, }; @@ -1353,6 +1354,7 @@ scheduler_set_operation_mode(scheduler_mode mode) for (int32_t i = 0; i < smp_get_num_cpus(); i++) acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + sCurrentModeID = mode; sCurrentMode = sSchedulerModes[mode]; sCurrentMode->switch_to_mode(); @@ -1539,11 +1541,7 @@ _scheduler_init() return result; } -#if 0 scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); -#else - scheduler_set_operation_mode(SCHEDULER_MODE_POWER_SAVING); -#endif add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); @@ -1663,3 +1661,16 @@ _user_estimate_max_scheduling_latency(thread_id id) return 2 * kMinThreadQuantum; } + +status_t +_user_set_scheduler_mode(int32 mode) +{ + return scheduler_set_operation_mode(static_cast(mode)); +} + + +int32 +_user_get_scheduler_mode(void) +{ + return sCurrentModeID; +} diff --git a/src/system/libroot/os/scheduler.c b/src/system/libroot/os/scheduler.c index 65e9713c16..dda2826595 100644 --- a/src/system/libroot/os/scheduler.c +++ b/src/system/libroot/os/scheduler.c @@ -60,3 +60,17 @@ estimate_max_scheduling_latency(thread_id thread) return _kern_estimate_max_scheduling_latency(thread); } + +status_t +set_scheduler_mode(int32 mode) +{ + return _kern_set_scheduler_mode(mode); +} + + +int32 +get_scheduler_mode(void) +{ + return _kern_get_scheduler_mode(); +} + From 6fb61d001a6022e27892cc643f345541c4e954dc Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 21 Nov 2013 00:59:40 +0100 Subject: [PATCH 131/273] scheduler: Preallocate temporary heaps for debug commands --- src/system/kernel/scheduler/low_latency.cpp | 1 + src/system/kernel/scheduler/scheduler.cpp | 68 +++++++++++++++------ 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index bc14cadb5c..fa534ad76f 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -91,6 +91,7 @@ should_rebalance(Thread* thread) CoreEntry* other = gCoreLoadHeap->PeekMinimum(); if (other == NULL) other = gCoreHighLoadHeap->PeekMinimum(); + ASSERT(other != NULL); return coreEntry->fLoad - other->fLoad >= kLoadDifference * 2; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index b8f738d695..e64330dda5 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -99,6 +99,10 @@ public: } // namespace Scheduler +static CPUHeap* sDebugCPUHeap; +static CoreLoadHeap* sDebugCoreHeap; +static PackageHeap* sDebugPackageHeap; + CPUEntry::CPUEntry() : @@ -248,8 +252,6 @@ dump_run_queue(int argc, char **argv) static void dump_heap(CPUHeap* heap) { - CPUHeap temp(smp_get_num_cpus()); - kprintf("cpu priority load\n"); CPUEntry* entry = heap->PeekMinimum(); while (entry) { @@ -259,17 +261,17 @@ dump_heap(CPUHeap* heap) gCPUEntries[cpu].fLoad / 10); heap->RemoveMinimum(); - temp.Insert(entry, key); + sDebugCPUHeap->Insert(entry, key); entry = heap->PeekMinimum(); } - entry = temp.PeekMinimum(); + entry = sDebugCPUHeap->PeekMinimum(); while (entry) { int32 key = CPUHeap::GetKey(entry); - temp.RemoveMinimum(); + sDebugCPUHeap->RemoveMinimum(); heap->Insert(entry, key); - entry = temp.PeekMinimum(); + entry = sDebugCPUHeap->PeekMinimum(); } } @@ -277,7 +279,6 @@ dump_heap(CPUHeap* heap) static void dump_core_load_heap(CoreLoadHeap* heap) { - CoreLoadHeap temp(gRunQueueCount); int32 cpuPerCore = smp_get_num_cpus() / gRunQueueCount; CoreEntry* entry = heap->PeekMinimum(); @@ -287,17 +288,17 @@ dump_core_load_heap(CoreLoadHeap* heap) entry->fLoad / cpuPerCore / 10); heap->RemoveMinimum(); - temp.Insert(entry, key); + sDebugCoreHeap->Insert(entry, key); entry = heap->PeekMinimum(); } - entry = temp.PeekMinimum(); + entry = sDebugCoreHeap->PeekMinimum(); while (entry) { int32 key = CoreLoadHeap::GetKey(entry); - temp.RemoveMinimum(); + sDebugCoreHeap->RemoveMinimum(); heap->Insert(entry, key); - entry = temp.PeekMinimum(); + entry = sDebugCoreHeap->PeekMinimum(); } } @@ -348,7 +349,6 @@ dump_idle_cores(int argc, char** argv) } else kprintf("No idle packages.\n"); - PackageHeap temp(smp_get_num_cpus()); kprintf("\nPackages with idle cores:\n"); PackageEntry* entry = gPackageUsageHeap->PeekMinimum(); @@ -374,17 +374,17 @@ dump_idle_cores(int argc, char** argv) kprintf("\n"); gPackageUsageHeap->RemoveMinimum(); - temp.Insert(entry, entry->fIdleCoreCount); + sDebugPackageHeap->Insert(entry, entry->fIdleCoreCount); entry = gPackageUsageHeap->PeekMinimum(); } - entry = temp.PeekMinimum(); + entry = sDebugPackageHeap->PeekMinimum(); while (entry != NULL) { int32 key = PackageHeap::GetKey(entry); - temp.RemoveMinimum(); + sDebugPackageHeap->RemoveMinimum(); gPackageUsageHeap->Insert(entry, key); - entry = temp.PeekMinimum(); + entry = sDebugPackageHeap->PeekMinimum(); } return 0; @@ -1429,6 +1429,29 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) } +static status_t +create_debug_heaps() +{ + sDebugCPUHeap = new(std::nothrow) CPUHeap(smp_get_num_cpus()); + if (sDebugCPUHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter cpuDeleter(sDebugCPUHeap); + + sDebugCoreHeap = new(std::nothrow) CoreLoadHeap(smp_get_num_cpus()); + if (sDebugCoreHeap == NULL) + return B_NO_MEMORY; + ObjectDeleter coreDeleter(sDebugCoreHeap); + + sDebugPackageHeap = new(std::nothrow) PackageHeap(smp_get_num_cpus()); + if (sDebugPackageHeap == NULL) + return B_NO_MEMORY; + + coreDeleter.Detach(); + cpuDeleter.Detach(); + return B_OK; +} + + static status_t _scheduler_init() { @@ -1488,7 +1511,7 @@ _scheduler_init() for (int32 i = 0; i < coreCount; i++) { gCoreEntries[i].fCoreID = i; - status_t result = gCoreLoadHeap->Insert(&gCoreEntries[i], 0); + result = gCoreLoadHeap->Insert(&gCoreEntries[i], 0); if (result != B_OK) return result; } @@ -1507,7 +1530,7 @@ _scheduler_init() if (gCPUPriorityHeaps[core].PeekMaximum() == NULL) gPackageEntries[package].fIdleCores.Insert(&gCoreEntries[core]); - status_t result + result = gCPUPriorityHeaps[core].Insert(&gCPUEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; @@ -1522,7 +1545,7 @@ _scheduler_init() return B_NO_MEMORY; ArrayDeleter pinnedRunQueuesDeleter(gPinnedRunQueues); for (int i = 0; i < cpuCount; i++) { - status_t result = gPinnedRunQueues[i].GetInitStatus(); + result = gPinnedRunQueues[i].GetInitStatus(); if (result != B_OK) return result; } @@ -1536,11 +1559,16 @@ _scheduler_init() return B_NO_MEMORY; ArrayDeleter runQueuesDeleter(gRunQueues); for (int i = 0; i < coreCount; i++) { - status_t result = gRunQueues[i].GetInitStatus(); + result = gRunQueues[i].GetInitStatus(); if (result != B_OK) return result; } + // create temporary heaps for debugging commands + result = create_debug_heaps(); + if (result != B_OK) + return result; + scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); add_debugger_command_etc("run_queue", &dump_run_queue, From 024541a4c8d35f9b3c5e27995b8f07be68d7c09a Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 21 Nov 2013 02:09:13 +0100 Subject: [PATCH 132/273] kernel: Improve rw_spinlock implementation * Add more debug checks * Reduce the number of executed instructions that lock the bus. --- headers/private/kernel/smp.h | 5 +- src/system/kernel/smp.cpp | 98 ++++++++++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 24 deletions(-) diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 8ef0e82a34..7690a82d64 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -100,8 +100,6 @@ release_spinlock_inline(spinlock* lock) #define acquire_spinlock(lock) acquire_spinlock_inline(lock) #define release_spinlock(lock) release_spinlock_inline(lock) -#endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION - static inline bool try_acquire_write_spinlock_inline(rw_spinlock* lock) @@ -207,4 +205,7 @@ release_read_seqlock_inline(seqlock* lock, uint32 count) { release_read_seqlock_inline(lock, count) +#endif // !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION + + #endif /* KERNEL_SMP_H */ diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index d73f787a8f..08203ef201 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -350,7 +350,7 @@ acquire_spinlock(spinlock* lock) #else while (1) { uint32 count = 0; - while (*lock != 0) { + while (atomic_get(lock) != 0) { if (++count == SPINLOCK_DEADLOCK_COUNT) { panic("acquire_spinlock(): Failed to acquire spinlock %p " "for a long time!", lock); @@ -404,7 +404,7 @@ acquire_spinlock_nocheck(spinlock *lock) #else while (1) { uint32 count = 0; - while (*lock != 0) { + while (atomic_get(lock) != 0) { if (++count == SPINLOCK_DEADLOCK_COUNT_NO_CHECK) { panic("acquire_spinlock(): Failed to acquire spinlock %p " "for a long time!", lock); @@ -447,7 +447,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) #else while (1) { uint32 count = 0; - while (*lock != 0) { + while (atomic_get(lock) != 0) { if (++count == SPINLOCK_DEADLOCK_COUNT) { panic("acquire_spinlock_cpu(): Failed to acquire spinlock " "%p for a long time!", lock); @@ -530,15 +530,31 @@ release_spinlock(spinlock *lock) bool try_acquire_write_spinlock(rw_spinlock* lock) { - return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; +#if DEBUG_SPINLOCKS + if (are_interrupts_enabled()) { + panic("try_acquire_write_spinlock: attempt to acquire lock %p with" + " interrupts enabled", lock); + } + + if (sNumCPUs < 2 && lock->lock != 0) { + panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " + "non-SMP system", lock); + } +#endif + + return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; } void acquire_write_spinlock(rw_spinlock* lock) { - if (sNumCPUs < 2) - return; +#if DEBUG_SPINLOCKS + if (are_interrupts_enabled()) { + panic("acquire_write_spinlock: attempt to acquire lock %p with" + " interrupts enabled", lock); + } +#endif uint32 count = 0; int currentCPU = smp_get_current_cpu(); @@ -546,14 +562,16 @@ acquire_write_spinlock(rw_spinlock* lock) if (try_acquire_write_spinlock(lock)) break; - if (++count == SPINLOCK_DEADLOCK_COUNT) { - panic("acquire_write_spinlock(): Failed to acquire spinlock %p " - "for a long time!", lock); - count = 0; - } + while (atomic_get(&lock->lock) != 0) { + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_write_spinlock(): Failed to acquire spinlock %p " + "for a long time!", lock); + count = 0; + } - process_all_pending_ici(currentCPU); - PAUSE(); + process_all_pending_ici(currentCPU); + PAUSE(); + } } } @@ -561,16 +579,37 @@ acquire_write_spinlock(rw_spinlock* lock) void release_write_spinlock(rw_spinlock* lock) { +#if DEBUG_SPINLOCKS + uint32 previous = atomic_get_and_set(&lock->lock, 0); + if ((previous & 1u << 31) == 0) { + panic("release_write_spinlock: lock %p was already released (value:" + " %x)\n", lock, previous); + } +#else atomic_set(&lock->lock, 0); +#endif } bool try_acquire_read_spinlock(rw_spinlock* lock) { +#if DEBUG_SPINLOCKS + if (are_interrupts_enabled()) { + panic("try_acquire_read_spinlock: attempt to acquire lock %p with" + " interrupts enabled", lock); + } + + if (sNumCPUs < 2 && lock->lock != 0) { + panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " + "non-SMP system", lock); + } +#endif + uint32 previous = atomic_add(&lock->lock, 1); if ((previous & (1 << 31)) == 0) return true; + atomic_test_and_set(&lock->lock, 1 << 31, previous); return false; } @@ -579,8 +618,12 @@ try_acquire_read_spinlock(rw_spinlock* lock) void acquire_read_spinlock(rw_spinlock* lock) { - if (sNumCPUs < 2) - return; +#if DEBUG_SPINLOCKS + if (are_interrupts_enabled()) { + panic("acquire_read_spinlock: attempt to acquire lock %p with" + " interrupts enabled", lock); + } +#endif uint32 count = 0; int currentCPU = smp_get_current_cpu(); @@ -588,14 +631,16 @@ acquire_read_spinlock(rw_spinlock* lock) if (try_acquire_read_spinlock(lock)) break; - if (++count == SPINLOCK_DEADLOCK_COUNT) { - panic("acquire_read_spinlock(): Failed to acquire spinlock %p " - "for a long time!", lock); - count = 0; - } + while (atomic_get(&lock->lock) != 0) { + if (++count == SPINLOCK_DEADLOCK_COUNT) { + panic("acquire_read_spinlock(): Failed to acquire spinlock %p " + "for a long time!", lock); + count = 0; + } - process_all_pending_ici(currentCPU); - PAUSE(); + process_all_pending_ici(currentCPU); + PAUSE(); + } } } @@ -603,7 +648,16 @@ acquire_read_spinlock(rw_spinlock* lock) void release_read_spinlock(rw_spinlock* lock) { +#if DEBUG_SPINLOCKS + uint32 previous = atomic_add(&lock->lock, -1); + if ((previous & 1 << 31) != 0) { + panic("release_read_spinlock: lock %p was already released (value:" + " %x)\n", lock, previous); + } +#else atomic_add(&lock->lock, -1); +#endif + } From 14e36af649464ae1bee6710b9ec27b196eca98ef Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 21 Nov 2013 03:24:54 +0100 Subject: [PATCH 133/273] kernel[/util]: Fix debug builds --- headers/private/kernel/util/Heap.h | 7 +++++++ headers/private/kernel/util/MinMaxHeap.h | 7 +++++++ src/system/kernel/scheduler/RunQueue.h | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h index ed7294c2b2..b2751eb6b3 100644 --- a/headers/private/kernel/util/Heap.h +++ b/headers/private/kernel/util/Heap.h @@ -346,5 +346,12 @@ HEAP_CLASS_NAME::_MoveDown(HeapLink* link) } +HEAP_TEMPLATE_LIST +Compare HEAP_CLASS_NAME::sCompare; + +HEAP_TEMPLATE_LIST +GetLink HEAP_CLASS_NAME::sGetLink; + + #endif // KERNEL_UTIL_HEAP_H diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 14efee5ac8..6e7caaea29 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -493,5 +493,12 @@ MIN_MAX_HEAP_CLASS_NAME::_RemoveLast(bool minTree) } +MIN_MAX_HEAP_TEMPLATE_LIST +Compare MIN_MAX_HEAP_CLASS_NAME::sCompare; + +MIN_MAX_HEAP_TEMPLATE_LIST +GetLink MIN_MAX_HEAP_CLASS_NAME::sGetLink; + + #endif // KERNEL_UTIL_MIN_MAX_HEAP_H diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index 9485a1c4f8..e33f1ab1bd 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -349,5 +349,12 @@ RUN_QUEUE_CLASS_NAME::GetConstIterator() const } +RUN_QUEUE_TEMPLATE_LIST +GetLink RUN_QUEUE_CLASS_NAME::sGetLink; + +RUN_QUEUE_TEMPLATE_LIST +GetLink RUN_QUEUE_CLASS_NAME::ConstIterator::sGetLink; + + #endif // RUN_QUEUE_H From 4ec76fd89d90e02bb4fc3d8d567234ab949ae9ed Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 21 Nov 2013 21:50:16 +0100 Subject: [PATCH 134/273] libroot: {g,s}et_scheduler_mode(): Avoid name clash Thanks Ingo for pointing this out. --- src/system/libroot/os/scheduler.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/system/libroot/os/scheduler.c b/src/system/libroot/os/scheduler.c index dda2826595..9bae2228b1 100644 --- a/src/system/libroot/os/scheduler.c +++ b/src/system/libroot/os/scheduler.c @@ -62,15 +62,19 @@ estimate_max_scheduling_latency(thread_id thread) status_t -set_scheduler_mode(int32 mode) +__set_scheduler_mode(int32 mode) { return _kern_set_scheduler_mode(mode); } int32 -get_scheduler_mode(void) +__get_scheduler_mode(void) { return _kern_get_scheduler_mode(); } + +B_DEFINE_WEAK_ALIAS(__set_scheduler_mode, set_scheduler_mode); +B_DEFINE_WEAK_ALIAS(__get_scheduler_mode, get_scheduler_mode); + From 65741c8b56b2fc74e8e1cd419b92f799e75f556d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 22 Nov 2013 03:00:08 +0100 Subject: [PATCH 135/273] scheduler: Improve locking --- src/system/kernel/scheduler/low_latency.cpp | 52 ++++++-- src/system/kernel/scheduler/power_saving.cpp | 90 ++++++++----- src/system/kernel/scheduler/scheduler.cpp | 122 ++++-------------- .../kernel/scheduler/scheduler_common.h | 15 ++- 4 files changed, 130 insertions(+), 149 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index fa534ad76f..9c264998ad 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -36,20 +36,43 @@ has_cache_expired(Thread* thread) } +static inline PackageEntry* +get_most_idle_package(void) +{ + PackageEntry* current = &gPackageEntries[0]; + for (int32 i = 1; i < gPackageCount; i++) { + if (gPackageEntries[i].fIdleCoreCount > current->fIdleCoreCount) + current = &gPackageEntries[i]; + } + + if (current->fIdleCoreCount == 0) + return NULL; + + return current; +} + + static int32 choose_core(Thread* thread) { - CoreEntry* entry; + CoreEntry* entry = NULL; - if (gIdlePackageList->Last() != NULL) { - // wake new package - PackageEntry* package = gIdlePackageList->Last(); - entry = package->fIdleCores.Last(); - } else if (gPackageUsageHeap->PeekMaximum() != NULL) { + SpinLocker locker(gIdlePackageLock); + // wake new package + PackageEntry* package = gIdlePackageList->Last(); + if (package == NULL) { // wake new core - PackageEntry* package = gPackageUsageHeap->PeekMaximum(); + package = get_most_idle_package(); + } + locker.Unlock(); + + if (package != NULL) { + SpinLocker _(package->fCoreLock); entry = package->fIdleCores.Last(); - } else { + } + + if (entry == NULL) { + ReadSpinLocker coreLocker(gCoreHeapsLock); // no idle cores, use least occupied core entry = gCoreLoadHeap->PeekMinimum(); if (entry == NULL) @@ -77,7 +100,7 @@ should_rebalance(Thread* thread) // If there is high load on this core but this thread does not contribute // significantly consider giving it to someone less busy. if (coreEntry->fLoad > kHighLoad) { - SpinLocker coreLocker(gCoreHeapsLock); + ReadSpinLocker coreLocker(gCoreHeapsLock); CoreEntry* other = gCoreLoadHeap->PeekMinimum(); if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference) @@ -86,7 +109,7 @@ should_rebalance(Thread* thread) // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... - SpinLocker coreLocker(gCoreHeapsLock); + ReadSpinLocker coreLocker(gCoreHeapsLock); CoreEntry* other = gCoreLoadHeap->PeekMinimum(); if (other == NULL) @@ -121,14 +144,17 @@ rebalance_irqs(bool idle) if (chosen == NULL || totalLoad < kLowLoad) return; - SpinLocker coreLocker(gCoreHeapsLock); + ReadSpinLocker coreLocker(gCoreHeapsLock); CoreEntry* other = gCoreLoadHeap->PeekMinimum(); if (other == NULL) other = gCoreHighLoadHeap->PeekMinimum(); - - int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; coreLocker.Unlock(); + SpinLocker cpuLocker(other->fCPULock); + int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; + cpuLocker.Unlock(); + + ASSERT(other != NULL); int32 thisCore = gCPUToCore[smp_get_current_cpu()]; diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index ac848e9ab0..d054f56575 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -41,6 +41,8 @@ switch_to_mode(void) static bool try_small_task_packing(Thread* thread) { + ReadSpinLocker locker(gCoreHeapsLock); + int32 core = sSmallTaskCore; return (core == -1 && gCoreLoadHeap->PeekMaximum() != NULL) || (core != -1 @@ -52,7 +54,9 @@ try_small_task_packing(Thread* thread) static int32 choose_small_task_core(void) { + ReadSpinLocker locker(gCoreHeapsLock); CoreEntry* candidate = gCoreLoadHeap->PeekMaximum(); + locker.Unlock(); if (candidate == NULL) return sSmallTaskCore; @@ -64,6 +68,32 @@ choose_small_task_core(void) } +static CoreEntry* +choose_idle_core(void) +{ + PackageEntry* current = NULL; + for (int32 i = 0; i < gPackageCount; i++) { + if (gPackageEntries[i].fIdleCoreCount != 0 && (current == NULL + || gPackageEntries[i].fIdleCoreCount + < current->fIdleCoreCount)) { + current = &gPackageEntries[i]; + } + } + + if (current == NULL) { + SpinLocker _(gIdlePackageLock); + current = gIdlePackageList->Last(); + } + + if (current != NULL) { + SpinLocker _(current->fCoreLock); + return current->fIdleCores.Last(); + } + + return NULL; +} + + static int32 choose_core(Thread* thread) { @@ -72,22 +102,22 @@ choose_core(Thread* thread) if (try_small_task_packing(thread)) { // try to pack all threads on one core entry = &gCoreEntries[choose_small_task_core()]; - } else if (gCoreLoadHeap->PeekMinimum() != NULL) { - // run immediately on already woken core - entry = gCoreLoadHeap->PeekMinimum(); - } else if (gPackageUsageHeap->PeekMinimum() != NULL) { - // wake new core - PackageEntry* package = gPackageUsageHeap->PeekMinimum(); - entry = package->fIdleCores.Last(); - } else if (gIdlePackageList->Last() != NULL) { - // wake new package - PackageEntry* package = gIdlePackageList->Last(); - entry = package->fIdleCores.Last(); } else { - // no idle cores, use least occupied core - entry = gCoreLoadHeap->PeekMinimum(); - if (entry == NULL) - entry = gCoreHighLoadHeap->PeekMinimum(); + ReadSpinLocker coreLocker(gCoreHeapsLock); + if (gCoreLoadHeap->PeekMinimum() != NULL) { + // run immediately on already woken core + entry = gCoreLoadHeap->PeekMinimum(); + } else { + coreLocker.Unlock(); + + entry = choose_idle_core(); + + coreLocker.Lock(); + if (entry == NULL) + entry = gCoreLoadHeap->PeekMinimum(); + if (entry == NULL) + entry = gCoreHighLoadHeap->PeekMinimum(); + } } ASSERT(entry != NULL); @@ -110,26 +140,26 @@ should_rebalance(Thread* thread) CoreEntry* coreEntry = &gCoreEntries[core]; if (coreEntry->fLoad > kHighLoad) { - SpinLocker coreLocker(gCoreHeapsLock); + ReadSpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { - CoreEntry* other = gCoreLoadHeap->PeekMaximum(); - - if (other == NULL) - sSmallTaskCore = -1; - else if (coreEntry->fLoad - schedulerThreadData->load < kHighLoad) + if (coreEntry->fLoad - schedulerThreadData->load < kHighLoad) return true; - else - sSmallTaskCore = other->fCoreID; + + choose_small_task_core(); return coreEntry->fLoad > kVeryHighLoad; } - CoreEntry* other = gCoreHighLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap->PeekMaximum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMaximum(); + other = gCoreHighLoadHeap->PeekMinimum(); + ASSERT(other != NULL); return coreEntry->fLoad - other->fLoad >= kLoadDifference / 2; } - return choose_small_task_core() != core; + int32 smallTaskCore = choose_small_task_core(); + if (smallTaskCore == -1) + return false; + return smallTaskCore != core; } @@ -144,7 +174,7 @@ pack_irqs(void) irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - SpinLocker coreLocker(gCoreHeapsLock); + ReadSpinLocker coreLocker(gCoreHeapsLock); int32 newCPU = gCPUPriorityHeaps[sSmallTaskCore].PeekMinimum()->fCPUNumber; coreLocker.Unlock(); @@ -185,12 +215,14 @@ rebalance_irqs(bool idle) if (chosen == NULL || chosen->load < kLowLoad) return; - SpinLocker coreLocker(gCoreHeapsLock); + ReadSpinLocker coreLocker(gCoreHeapsLock); CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + coreLocker.Unlock(); if (other == NULL) return; + SpinLocker cpuLocker(other->fCPULock); int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; - coreLocker.Unlock(); + cpuLocker.Unlock(); int32 thisCore = gCPUToCore[smp_get_current_cpu()]; if (other->fCoreID == thisCore) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index e64330dda5..ecd8f7d0c6 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -60,12 +60,12 @@ CPUHeap* gCPUPriorityHeaps; CoreEntry* gCoreEntries; CoreLoadHeap* gCoreLoadHeap; CoreLoadHeap* gCoreHighLoadHeap; -spinlock gCoreHeapsLock = B_SPINLOCK_INITIALIZER; +rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; PackageEntry* gPackageEntries; -PackageHeap* gPackageUsageHeap; IdlePackageList* gIdlePackageList; -spinlock gIdlePackageLock = B_SPINLOCK_INITIALIZER; +spinlock gIdlePackageLock; +int32 gPackageCount = B_SPINLOCK_INITIALIZER; ThreadRunQueue* gRunQueues; ThreadRunQueue* gPinnedRunQueues; @@ -101,7 +101,6 @@ public: static CPUHeap* sDebugCPUHeap; static CoreLoadHeap* sDebugCoreHeap; -static PackageHeap* sDebugPackageHeap; CPUEntry::CPUEntry() @@ -120,7 +119,8 @@ CoreEntry::CoreEntry() fActiveTime(0), fLoad(0) { - B_INITIALIZE_SPINLOCK(&fLock); + B_INITIALIZE_SPINLOCK(&fCPULock); + B_INITIALIZE_SPINLOCK(&fQueueLock); } @@ -129,6 +129,7 @@ PackageEntry::PackageEntry() fIdleCoreCount(0), fCoreCount(0) { + B_INITIALIZE_SPINLOCK(&fCoreLock); } @@ -349,44 +350,6 @@ dump_idle_cores(int argc, char** argv) } else kprintf("No idle packages.\n"); - kprintf("\nPackages with idle cores:\n"); - - PackageEntry* entry = gPackageUsageHeap->PeekMinimum(); - if (entry == NULL) - kprintf("No packages.\n"); - else - kprintf("package count cores\n"); - - while (entry != NULL) { - kprintf("%-7" B_PRId32 " %-5" B_PRId32 " ", entry->fPackageID, - entry->fIdleCoreCount); - - DoublyLinkedList::ReverseIterator iterator - = entry->fIdleCores.GetReverseIterator(); - if (iterator.HasNext()) { - while (iterator.HasNext()) { - CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, - iterator.HasNext() ? ", " : ""); - } - } else - kprintf("-"); - kprintf("\n"); - - gPackageUsageHeap->RemoveMinimum(); - sDebugPackageHeap->Insert(entry, entry->fIdleCoreCount); - - entry = gPackageUsageHeap->PeekMinimum(); - } - - entry = sDebugPackageHeap->PeekMinimum(); - while (entry != NULL) { - int32 key = PackageHeap::GetKey(entry); - sDebugPackageHeap->RemoveMinimum(); - gPackageUsageHeap->Insert(entry, key); - entry = sDebugPackageHeap->PeekMinimum(); - } - return 0; } @@ -437,7 +400,7 @@ update_load_heaps(int32 core) CoreEntry* entry = &gCoreEntries[core]; - SpinLocker coreLocker(gCoreHeapsLock); + WriteSpinLocker coreLocker(gCoreHeapsLock); int32 cpuPerCore = smp_get_num_cpus() / gRunQueueCount; int32 newKey = entry->fLoad / cpuPerCore; @@ -512,6 +475,8 @@ update_cpu_priority(int32 cpu, int32 priority) { int32 core = gCPUToCore[cpu]; + SpinLocker coreLocker(gCoreEntries[core].fCPULock); + int32 corePriority = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); gCPUEntries[cpu].fPriority = priority; @@ -529,7 +494,7 @@ update_cpu_priority(int32 cpu, int32 priority) int32 package = gCPUToPackage[cpu]; PackageEntry* packageEntry = &gPackageEntries[package]; if (maxPriority == B_IDLE_PRIORITY) { - SpinLocker _(gIdlePackageLock); + SpinLocker _(packageEntry->fCoreLock); // core goes idle ASSERT(packageEntry->fIdleCoreCount >= 0); @@ -538,27 +503,13 @@ update_cpu_priority(int32 cpu, int32 priority) packageEntry->fIdleCoreCount++; packageEntry->fIdleCores.Add(&gCoreEntries[core]); - if (packageEntry->fIdleCoreCount == 1) { - // first core on that package to go idle - - if (packageEntry->fCoreCount > 1) - gPackageUsageHeap->Insert(packageEntry, 1); - else - gIdlePackageList->Add(packageEntry); - } else if (packageEntry->fIdleCoreCount - == packageEntry->fCoreCount) { + if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { // package goes idle - gPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(gPackageUsageHeap->PeekMinimum() == packageEntry); - gPackageUsageHeap->RemoveMinimum(); - + SpinLocker _(gIdlePackageLock); gIdlePackageList->Add(packageEntry); - } else { - gPackageUsageHeap->ModifyKey(packageEntry, - packageEntry->fIdleCoreCount); } } else if (corePriority == B_IDLE_PRIORITY) { - SpinLocker _(gIdlePackageLock); + SpinLocker _(packageEntry->fCoreLock); // core wakes up ASSERT(packageEntry->fIdleCoreCount > 0); @@ -569,20 +520,8 @@ update_cpu_priority(int32 cpu, int32 priority) if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { // package wakes up + SpinLocker _(gIdlePackageLock); gIdlePackageList->Remove(packageEntry); - - if (packageEntry->fIdleCoreCount > 0) { - gPackageUsageHeap->Insert(packageEntry, - packageEntry->fIdleCoreCount); - } - } else if (packageEntry->fIdleCoreCount == 0) { - // no more idle cores in the package - gPackageUsageHeap->ModifyKey(packageEntry, 0); - ASSERT(gPackageUsageHeap->PeekMinimum() == packageEntry); - gPackageUsageHeap->RemoveMinimum(); - } else { - gPackageUsageHeap->ModifyKey(packageEntry, - packageEntry->fIdleCoreCount); } } } @@ -599,6 +538,7 @@ choose_core(Thread* thread) static inline int32 choose_cpu(int32 core) { + SpinLocker cpuLocker(gCoreEntries[core].fCPULock); CPUEntry* entry = gCPUPriorityHeaps[core].PeekMinimum(); ASSERT(entry != NULL); return entry->fCPUNumber; @@ -608,8 +548,6 @@ choose_cpu(int32 core) static bool choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) { - SpinLocker coreLocker(gCoreHeapsLock); - if (targetCore == -1 && targetCPU != -1) targetCore = gCPUToCore[targetCPU]; else if (targetCore != -1 && targetCPU == -1) @@ -779,7 +717,7 @@ enqueue(Thread* thread, bool newOne) TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", thread->id, threadPriority, targetCPU, targetCore); - SpinLocker runQueueLocker(gCoreEntries[targetCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[targetCore].fQueueLock); thread->scheduler_data->enqueued = true; if (pinned) gPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); @@ -832,7 +770,7 @@ put_back(Thread* thread) int32 core = gCPUToCore[smp_get_current_cpu()]; - SpinLocker runQueueLocker(gCoreEntries[core].fLock); + SpinLocker runQueueLocker(gCoreEntries[core].fQueueLock); thread->scheduler_data->enqueued = true; if (thread->pinned_to_cpu > 0) { int32 pinnedCPU = thread->previous_cpu->cpu_num; @@ -875,10 +813,8 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) cancel_penalty(thread); thread->priority = priority; - if (thread->state == B_THREAD_RUNNING) { - SpinLocker coreLocker(gCoreHeapsLock); + if (thread->state == B_THREAD_RUNNING) update_cpu_priority(thread->cpu->cpu_num, priority); - } return oldPriority; } @@ -890,7 +826,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) int32 previougCore = thread->scheduler_data->previous_core; ASSERT(previougCore >= 0); - SpinLocker runQueueLocker(gCoreEntries[previougCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[previougCore].fQueueLock); // the thread might have been already dequeued and is about to start // running once we release its scheduler_lock, in such case we can not @@ -1022,7 +958,7 @@ choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) { int32 thisCore = gCPUToCore[thisCPU]; - SpinLocker runQueueLocker(gCoreEntries[thisCore].fLock); + SpinLocker runQueueLocker(gCoreEntries[thisCore].fQueueLock); Thread* sharedThread = gRunQueues[thisCore].PeekMaximum(); Thread* pinnedThread = gPinnedRunQueues[thisCPU].PeekMaximum(); @@ -1234,10 +1170,7 @@ _scheduler_reschedule(void) oldThread, nextThread); // update CPU heap - { - SpinLocker coreLocker(gCoreHeapsLock); - update_cpu_priority(thisCPU, get_effective_priority(nextThread)); - } + update_cpu_priority(thisCPU, get_effective_priority(nextThread)); nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; @@ -1440,13 +1373,7 @@ create_debug_heaps() sDebugCoreHeap = new(std::nothrow) CoreLoadHeap(smp_get_num_cpus()); if (sDebugCoreHeap == NULL) return B_NO_MEMORY; - ObjectDeleter coreDeleter(sDebugCoreHeap); - sDebugPackageHeap = new(std::nothrow) PackageHeap(smp_get_num_cpus()); - if (sDebugPackageHeap == NULL) - return B_NO_MEMORY; - - coreDeleter.Detach(); cpuDeleter.Detach(); return B_OK; } @@ -1463,6 +1390,7 @@ _scheduler_init() return result; gRunQueueCount = coreCount; gSingleCore = coreCount == 1; + gPackageCount = packageCount; // create package heap and idle package stack gPackageEntries = new(std::nothrow) PackageEntry[packageCount]; @@ -1470,11 +1398,6 @@ _scheduler_init() return B_NO_MEMORY; ArrayDeleter packageEntriesDeleter(gPackageEntries); - gPackageUsageHeap = new(std::nothrow) PackageHeap(packageCount); - if (gPackageUsageHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter packageHeapDeleter(gPackageUsageHeap); - gIdlePackageList = new(std::nothrow) IdlePackageList; if (gIdlePackageList == NULL) return B_NO_MEMORY; @@ -1589,7 +1512,6 @@ _scheduler_init() coreEntriesDeleter.Detach(); cpuEntriesDeleter.Detach(); packageEntriesDeleter.Detach(); - packageHeapDeleter.Detach(); packageListDeleter.Detach(); return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index ea3aa422ad..7340e93320 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -79,7 +79,8 @@ struct CoreEntry : public MinMaxHeapLinkImpl, int32 fCoreID; - spinlock fLock; + spinlock fCPULock; + spinlock fQueueLock; bigtime_t fStartedBottom; bigtime_t fReachedBottom; @@ -95,9 +96,9 @@ typedef MinMaxHeap CoreLoadHeap; extern CoreEntry* gCoreEntries; extern CoreLoadHeap* gCoreLoadHeap; extern CoreLoadHeap* gCoreHighLoadHeap; -extern spinlock gCoreHeapsLock; +extern rw_spinlock gCoreHeapsLock; -// sPackageUsageHeap is used to decide which core should be woken up from the +// gPackageEntries are used to decide which core should be woken up from the // idle state. When aiming for performance we should use as many packages as // possible with as little cores active in each package as possible (so that the // package can enter any boost mode if it has one and the active core have more @@ -106,24 +107,24 @@ extern spinlock gCoreHeapsLock; // packages can go to the deep state of sleep). The heap stores only packages // with at least one core active and one core idle. The packages with all cores // idle are stored in sPackageIdleList (in LIFO manner). -struct PackageEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { +struct PackageEntry : public DoublyLinkedListLinkImpl { PackageEntry(); int32 fPackageID; + spinlock fCoreLock; + DoublyLinkedList fIdleCores; int32 fIdleCoreCount; int32 fCoreCount; } CACHE_LINE_ALIGN; -typedef MinMaxHeap PackageHeap; typedef DoublyLinkedList IdlePackageList; extern PackageEntry* gPackageEntries; -extern PackageHeap* gPackageUsageHeap; extern IdlePackageList* gIdlePackageList; extern spinlock gIdlePackageLock; +extern int32 gPackageCount; // The run queues. Holds the threads ready to run ordered by priority. // One queue per schedulable target per core. Additionally, each From 1a7eb50254e1cee8160de17253aa6972c3d960cf Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 22 Nov 2013 03:20:19 +0100 Subject: [PATCH 136/273] scheduler: Calculate correct load on SMT processors --- src/system/kernel/scheduler/low_latency.cpp | 16 +++++++++++----- src/system/kernel/scheduler/power_saving.cpp | 15 +++++++++------ src/system/kernel/scheduler/scheduler.cpp | 10 +++++----- src/system/kernel/scheduler/scheduler_common.h | 9 +++++++++ 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 9c264998ad..5f1224c45e 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -92,19 +92,23 @@ should_rebalance(Thread* thread) CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; + int32 coreLoad = get_core_load(coreEntry); + // If the thread produces more than 50% of the load, leave it here. In // such situation it is better to move other threads away. - if (schedulerThreadData->load >= coreEntry->fLoad / 2) + if (schedulerThreadData->load >= coreLoad / 2) return false; // If there is high load on this core but this thread does not contribute // significantly consider giving it to someone less busy. - if (coreEntry->fLoad > kHighLoad) { + if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); CoreEntry* other = gCoreLoadHeap->PeekMinimum(); - if (other != NULL && coreEntry->fLoad - other->fLoad >= kLoadDifference) + if (other != NULL && coreLoad - get_core_load(other) + >= kLoadDifference) { return true; + } } // No cpu bound threads - the situation is quite good. Make sure it @@ -115,7 +119,7 @@ should_rebalance(Thread* thread) if (other == NULL) other = gCoreHighLoadHeap->PeekMinimum(); ASSERT(other != NULL); - return coreEntry->fLoad - other->fLoad >= kLoadDifference * 2; + return coreLoad - get_core_load(other) >= kLoadDifference * 2; } @@ -161,8 +165,10 @@ rebalance_irqs(bool idle) if (other->fCoreID == thisCore) return; - if (other->fLoad + kLoadDifference >= gCoreEntries[thisCore].fLoad) + if (get_core_load(other) + kLoadDifference + >= get_core_load(&gCoreEntries[thisCore])) { return; + } assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index d054f56575..51485eee75 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -46,7 +46,7 @@ try_small_task_packing(Thread* thread) int32 core = sSmallTaskCore; return (core == -1 && gCoreLoadHeap->PeekMaximum() != NULL) || (core != -1 - && gCoreEntries[core].fLoad + thread->scheduler_data->load + && get_core_load(&gCoreEntries[core]) + thread->scheduler_data->load < kHighLoad); } @@ -139,21 +139,22 @@ should_rebalance(Thread* thread) int32 core = schedulerThreadData->previous_core; CoreEntry* coreEntry = &gCoreEntries[core]; - if (coreEntry->fLoad > kHighLoad) { + int32 coreLoad = get_core_load(coreEntry); + if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { - if (coreEntry->fLoad - schedulerThreadData->load < kHighLoad) + if (coreLoad - schedulerThreadData->load < kHighLoad) return true; choose_small_task_core(); - return coreEntry->fLoad > kVeryHighLoad; + return coreLoad > kVeryHighLoad; } CoreEntry* other = gCoreLoadHeap->PeekMaximum(); if (other == NULL) other = gCoreHighLoadHeap->PeekMinimum(); ASSERT(other != NULL); - return coreEntry->fLoad - other->fLoad >= kLoadDifference / 2; + return coreLoad - get_core_load(other) >= kLoadDifference / 2; } int32 smallTaskCore = choose_small_task_core(); @@ -228,8 +229,10 @@ rebalance_irqs(bool idle) if (other->fCoreID == thisCore) return; - if (other->fLoad + kLoadDifference >= gCoreEntries[thisCore].fLoad) + if (get_core_load(other) + kLoadDifference + >= get_core_load(&gCoreEntries[thisCore])) { return; + } assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ecd8f7d0c6..f3fe8d7cd1 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -116,6 +116,7 @@ CPUEntry::CPUEntry() CoreEntry::CoreEntry() : + fCPUCount(0), fActiveTime(0), fLoad(0) { @@ -280,13 +281,11 @@ dump_heap(CPUHeap* heap) static void dump_core_load_heap(CoreLoadHeap* heap) { - int32 cpuPerCore = smp_get_num_cpus() / gRunQueueCount; - CoreEntry* entry = heap->PeekMinimum(); while (entry) { int32 key = CoreLoadHeap::GetKey(entry); kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, - entry->fLoad / cpuPerCore / 10); + get_core_load(entry) / 10); heap->RemoveMinimum(); sDebugCoreHeap->Insert(entry, key); @@ -402,8 +401,7 @@ update_load_heaps(int32 core) WriteSpinLocker coreLocker(gCoreHeapsLock); - int32 cpuPerCore = smp_get_num_cpus() / gRunQueueCount; - int32 newKey = entry->fLoad / cpuPerCore; + int32 newKey = get_core_load(entry); int32 oldKey = CoreLoadHeap::GetKey(entry); ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); @@ -1062,6 +1060,7 @@ update_cpu_performance(Thread* thread, int32 thisCore) { int32 load = max_c(thread->scheduler_data->load, gCoreEntries[thisCore].fLoad); + load /= gCoreEntries[thisCore].fCPUCount; load = min_c(max_c(load, 0), kMaxLoad); if (load < kTargetLoad) { @@ -1433,6 +1432,7 @@ _scheduler_init() for (int32 i = 0; i < coreCount; i++) { gCoreEntries[i].fCoreID = i; + gCoreEntries[i].fCPUCount = cpuCount / coreCount; result = gCoreLoadHeap->Insert(&gCoreEntries[i], 0); if (result != B_OK) diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 7340e93320..887060368e 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -79,6 +79,8 @@ struct CoreEntry : public MinMaxHeapLinkImpl, int32 fCoreID; + int32 fCPUCount; + spinlock fCPULock; spinlock fQueueLock; @@ -174,6 +176,13 @@ struct scheduler_thread_data { }; +static inline int32 +get_core_load(struct Scheduler::CoreEntry* core) +{ + return core->fLoad / core->fCPUCount; +} + + /*! Switches the currently running thread. This is a service function for scheduler implementations. From 7c7b9b795bbb537b889e96e335dd9a4add8b2789 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 22 Nov 2013 04:25:37 +0100 Subject: [PATCH 137/273] scheduler: Return time stolen by irq and ipi handlers --- src/system/kernel/scheduler/scheduler.cpp | 9 +++++++++ src/system/kernel/scheduler/scheduler_common.h | 1 + 2 files changed, 10 insertions(+) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index f3fe8d7cd1..105d7fa13e 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1101,6 +1101,11 @@ _scheduler_reschedule(void) oldThread->state = oldThread->next_state; scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; + // return time spent in interrupts + schedulerOldThreadData->stolen_time + += gCPU[thisCPU].interrupt_time + - schedulerOldThreadData->last_interrupt_time; + bool enqueueOldThread = false; bool putOldThreadAtBack = false; switch (oldThread->next_state) { @@ -1184,6 +1189,10 @@ _scheduler_reschedule(void) // track CPU activity track_cpu_activity(oldThread, nextThread, thisCore); + // start counting time spent in interrupts + nextThread->scheduler_data->last_interrupt_time + = gCPU[thisCPU].interrupt_time; + if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; if (!oldThread->cpu->preempted) diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 887060368e..190e968978 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -162,6 +162,7 @@ struct scheduler_thread_data { bigtime_t time_left; bigtime_t stolen_time; bigtime_t quantum_start; + bigtime_t last_interrupt_time; bigtime_t measure_active_time; bigtime_t measure_time; From 99a2e56579b306d366d0f390f45e5b7e69b2be34 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 22 Nov 2013 05:02:34 +0100 Subject: [PATCH 138/273] drivers: Remove Intel Enhanced SpeedStep driver In current state this driver is useless anyway. It's not MP safe. The P-state values stored in hardcoded tables are only for processors on which we can't use SpeedStep, because of lack of invariant TSC. Proper driver should get P-states from ACPI, ensure that the CPU offers invariant TSC and obviouslt be MP safe. --- build/jam/packages/Haiku | 1 - build/jam/packages/HaikuBootstrap | 1 - src/add-ons/kernel/drivers/power/Jamfile | 1 - .../drivers/power/enhanced_speedstep/Jamfile | 18 - .../enhanced_speedstep/enhanced_speedstep.cpp | 407 --------- .../enhanced_speedstep/enhanced_speedstep.h | 29 - .../power/enhanced_speedstep/frequency.cpp | 202 ----- .../power/enhanced_speedstep/frequency.h | 852 ------------------ 8 files changed, 1511 deletions(-) delete mode 100644 src/add-ons/kernel/drivers/power/enhanced_speedstep/Jamfile delete mode 100644 src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp delete mode 100644 src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h delete mode 100644 src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.cpp delete mode 100644 src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.h diff --git a/build/jam/packages/Haiku b/build/jam/packages/Haiku index 2ca840a6ff..46df98aee6 100644 --- a/build/jam/packages/Haiku +++ b/build/jam/packages/Haiku @@ -61,7 +61,6 @@ if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 { # drivers AddNewDriversToPackage disk scsi : scsi_cd scsi_disk ; AddNewDriversToPackage disk virtual : virtio_block ; -AddNewDriversToPackage power : enhanced_speedstep@x86 ; AddNewDriversToPackage power : acpi_battery@x86 ; #AddNewDriversToPackage display : display_controls@x86 ; diff --git a/build/jam/packages/HaikuBootstrap b/build/jam/packages/HaikuBootstrap index f4a419d7d7..1fafbb20fe 100644 --- a/build/jam/packages/HaikuBootstrap +++ b/build/jam/packages/HaikuBootstrap @@ -60,7 +60,6 @@ if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 { # drivers AddNewDriversToPackage disk scsi : scsi_cd scsi_disk ; AddNewDriversToPackage disk virtual : virtio_block ; -AddNewDriversToPackage power : enhanced_speedstep@x86 ; AddNewDriversToPackage power : acpi_battery@x86 ; #AddNewDriversToPackage display : display_controls@x86 ; diff --git a/src/add-ons/kernel/drivers/power/Jamfile b/src/add-ons/kernel/drivers/power/Jamfile index 45feced8e9..2755d26fed 100644 --- a/src/add-ons/kernel/drivers/power/Jamfile +++ b/src/add-ons/kernel/drivers/power/Jamfile @@ -4,5 +4,4 @@ SubInclude HAIKU_TOP src add-ons kernel drivers power acpi_battery ; SubInclude HAIKU_TOP src add-ons kernel drivers power acpi_button ; SubInclude HAIKU_TOP src add-ons kernel drivers power acpi_lid ; SubInclude HAIKU_TOP src add-ons kernel drivers power acpi_thermal ; -SubInclude HAIKU_TOP src add-ons kernel drivers power enhanced_speedstep ; SubInclude HAIKU_TOP src add-ons kernel drivers power x86_cpuidle ; diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/Jamfile b/src/add-ons/kernel/drivers/power/enhanced_speedstep/Jamfile deleted file mode 100644 index 81878167d9..0000000000 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/Jamfile +++ /dev/null @@ -1,18 +0,0 @@ -SubDir HAIKU_TOP src add-ons kernel drivers power enhanced_speedstep ; - -SetSubDirSupportedPlatformsBeOSCompatible ; - -if $(TARGET_PLATFORM) != haiku { - # Needed for . Unfortunately we also get the other headers there, - # that we don't really want. - UsePublicHeaders drivers ; -} - -UsePrivateHeaders kernel ; - -KernelAddon enhanced_speedstep : - enhanced_speedstep.cpp - frequency.cpp - ; - -Depends acpi_enhanced_speedstep : acpi ; \ No newline at end of file diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp b/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp deleted file mode 100644 index eaf29c2850..0000000000 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include "enhanced_speedstep.h" -#include - -#include "frequency.h" - - -#define EST_MODULE_NAME "drivers/power/enhanced_speedstep/driver_v1" - -#define EST_DEVICE_MODULE_NAME "drivers/power/enhanced_speedstep/device_v1" - -/* Base Namespace devices are published to */ -#define EST_BASENAME "power/enhanced_speedstep/%d" - -// name of pnp generator of path ids -#define EST_PATHID_GENERATOR "enhanced_speedstep/path_id" - -static device_manager_info *sDeviceManager; -static ConditionVariable sFrequencyCondition; -static int32 sCurrentID; - - -static status_t -est_read(void* _cookie, off_t position, void *buffer, size_t* numBytes) -{ - if (*numBytes < 1) - return B_IO_ERROR; - - est_cookie *device = (est_cookie *)_cookie; - - if (position == 0) { - size_t max_len = *numBytes; - char *str = (char *)buffer; - - snprintf(str, max_len, "CPU Frequency states:\n"); - max_len-= strlen(str); - str += strlen(str); - - freq_info *freqsInfo = device->available_states; - freq_info *f; - for (f = freqsInfo; f->frequency != 0; f++) { - snprintf(str, max_len, " Frequency %hu, Volts %hu, Power %i, " - "Latency %i, id %hu\n", f->frequency, f->volts, f->power, f->id, - EST_TRANS_LAT); - max_len-= strlen(str); - str += strlen(str); - } - - freq_info *f2 = est_get_current(freqsInfo); - if (f2) { - snprintf(str, max_len, "\nCurrent State: Frequency %hu, Volts %hu, " - "Power %i, Latency %i\n", f2->frequency, f2->volts, f2->power, - EST_TRANS_LAT); - } - - *numBytes = strlen((char *)buffer); - } else { - *numBytes = 0; - } - - return B_OK; -} - - -static status_t -est_write(void* cookie, off_t position, const void* buffer, size_t* numBytes) -{ - return B_ERROR; -} - - -status_t -est_control(void* _cookie, uint32 op, void* arg, size_t len) -{ - est_cookie* device = (est_cookie*)_cookie; - status_t err = B_ERROR; - - uint32* magicId; - uint16* id; - freq_info* freqInfo = NULL; - switch (op) { - case IDENTIFY_DEVICE: - if (len < sizeof(uint32)) - return B_IO_ERROR; - magicId = (uint32*)arg; - *magicId = kMagicFreqID; - err = B_OK; - break; - - case GET_CPU_FREQ_STATES: - if (len < sizeof(freq_info) * (device->number_states + 1)) - return B_IO_ERROR; - freqInfo = (freq_info*)arg; - user_memcpy(freqInfo, device->available_states, - sizeof(freq_info) * (device->number_states + 1)); - err = B_OK; - break; - - case GET_CURENT_CPU_FREQ_STATE: - if (len < sizeof(uint16)) - return B_IO_ERROR; - freqInfo = est_get_current(device->available_states); - if (!freqInfo) - return B_ERROR; - atomic_set(&sCurrentID, freqInfo->id); - *((uint16*)arg) = freqInfo->id; - err = B_OK; - break; - - case SET_CPU_FREQ_STATE: - if (len < sizeof(uint16)) - return B_IO_ERROR; - id = (uint16*)arg; - err = est_set_id16(*id); - if (err == B_OK) { - atomic_set(&sCurrentID, *id); - sFrequencyCondition.NotifyAll(); - } - break; - - case WATCH_CPU_FREQ: - if (len < sizeof(uint16)) - return B_IO_ERROR; - sFrequencyCondition.Wait(); - if (atomic_get(&(device->stop_watching))) { - atomic_set(&(device->stop_watching), 0); - err = B_ERROR; - } else { - *((uint16*)arg) = atomic_get(&sCurrentID); - err = B_OK; - } - break; - - case STOP_WATCHING_CPU_FREQ: - atomic_set(&(device->stop_watching), 1); - sFrequencyCondition.NotifyAll(); - err = B_OK; - break; - } - return err; -} - - -static status_t -est_open(void *initCookie, const char *path, int flags, void** cookie) -{ - TRACE("est: open\n"); - est_cookie *device; - device = (est_cookie *)calloc(1, sizeof(est_cookie)); - if (device == NULL) - return B_NO_MEMORY; - - *cookie = device; - - device_node *node = (device_node *)initCookie; - device->node = node; - - device_node *parent; - parent = sDeviceManager->get_parent_node(node); - sDeviceManager->get_driver(parent, (driver_module_info **)&device->acpi, - (void **)&device->acpi_cookie); - sDeviceManager->put_node(parent); - - device->stop_watching = 0; - - // enable enhanced speedstep - uint64 msrMisc = x86_read_msr(MSR_MISC); - if ((msrMisc & MSR_EST_ENABLED) == 0) { - TRACE("est: enable enhanced speedstep\n"); - x86_write_msr(MSR_MISC, msrMisc | MSR_EST_ENABLED); - - uint64 msrMisc = x86_read_msr(MSR_MISC); - if ((msrMisc & MSR_EST_ENABLED) == 0) { - TRACE("est: enable enhanced speedstep failed\n"); - return B_ERROR; - } - } - - // get freq_info - if (est_get_info(&(device->available_states)) != B_OK) - return B_ERROR; - freq_info *freqsInfo = device->available_states; - - // count number of states - TRACE("est: frequency info:\n"); - freq_info *f; - device->number_states = 0; - for (f = freqsInfo; f->frequency != 0; f++) { - TRACE("est: Frequency %u, Volts %u, Power %i, Latency %u, id %u\n", - f->frequency, f->volts, f->power, f->id, EST_TRANS_LAT); - device->number_states++; - } - - // print current frequency - freq_info *f2 = est_get_current(freqsInfo); - if (f2) { - TRACE("est: Current Frequency %u, Volts %u, Power %i, Latency %u\n", - f2->frequency, f2->volts, f2->power, EST_TRANS_LAT); - } - - return B_OK; -} - - -static status_t -est_close(void* cookie) -{ - est_cookie *device = (est_cookie*)cookie; - free(device); - return B_OK; -} - - -static status_t -est_free(void* cookie) -{ - return B_OK; -} - - -// #pragma mark - driver module API - - -static float -est_support(device_node *parent) -{ - // make sure parent is really the ACPI bus manager - const char *bus; - if (sDeviceManager->get_attr_string(parent, B_DEVICE_BUS, &bus, false)) - return -1; - - if (strcmp(bus, "acpi")) - return 0.0; - - // check whether it's really a cpu Device - uint32 deviceType; - if (sDeviceManager->get_attr_uint32(parent, ACPI_DEVICE_TYPE_ITEM, - &deviceType, false) != B_OK - || deviceType != ACPI_TYPE_PROCESSOR) { - return 0.0; - } - - // check if cpu support est - uint32 cpuNum = 0; - system_info sysInfo; - if (get_system_info(&sysInfo) != B_OK) - return 0.0; - TRACE("est: cpu_type: %u vendor %u model %u\n", sysInfo.cpu_type, - sysInfo.cpu_type & B_CPU_x86_VENDOR_MASK, sysInfo.cpu_type & 0x00FF); - if ((sysInfo.cpu_type & B_CPU_x86_VENDOR_MASK) != B_CPU_INTEL_x86) - return 0.0; - - // TODO: Make the code SMP safe! - if (sysInfo.cpu_count > 1) - return 0.0; - - cpuid_info info; - if (get_cpuid(&info, 1, cpuNum) != B_OK) - return 0.0; - - TRACE("est: extended_features: %i\n", int(info.eax_1.extended_features)); - - // check for enhanced speedstep - if ((info.eax_1.extended_features & IA32_FEATURE_EXT_EST) == 0) - return 0.0; - - TRACE("est: supported\n"); - return 0.6; -} - - -static status_t -est_register_device(device_node *node) -{ - device_attr attrs[] = { - { B_DEVICE_PRETTY_NAME, B_STRING_TYPE, - { string: "ACPI Enhanced Speedstep" }}, - { NULL } - }; - - return sDeviceManager->register_node(node, EST_MODULE_NAME, attrs, - NULL, NULL); -} - - -static status_t -est_init_driver(device_node *node, void **_driverCookie) -{ - *_driverCookie = node; - - sFrequencyCondition.Init(NULL, "frequency cv"); - sCurrentID = -1; - - return B_OK; -} - - -static void -est_uninit_driver(void *driverCookie) -{ -} - - -static status_t -est_register_child_devices(void *_cookie) -{ - device_node *node = (device_node*)_cookie; - - int pathID = sDeviceManager->create_id(EST_PATHID_GENERATOR); - if (pathID < 0) { - TRACE("est_register_child_devices: couldn't create a path_id\n"); - return B_ERROR; - } - - char name[128]; - snprintf(name, sizeof(name), EST_BASENAME, pathID); - - return sDeviceManager->publish_device(node, name, EST_DEVICE_MODULE_NAME); -} - - -static status_t -est_init_device(void *driverCookie, void **cookie) -{ - // driverCookie is the device node - *cookie = driverCookie; - return B_OK; -} - - -static void -est_uninit_device(void *_cookie) -{ - -} - - - -module_dependency module_dependencies[] = { - { B_DEVICE_MANAGER_MODULE_NAME, (module_info **)&sDeviceManager }, - {} -}; - - -driver_module_info est_driver_module = { - { - EST_MODULE_NAME, - 0, - NULL - }, - - est_support, - est_register_device, - est_init_driver, - est_uninit_driver, - est_register_child_devices, - NULL, // rescan - NULL, // removed -}; - - -struct device_module_info est_device_module = { - { - EST_DEVICE_MODULE_NAME, - 0, - NULL - }, - - est_init_device, - est_uninit_device, - NULL, - - est_open, - est_close, - est_free, - est_read, - est_write, - NULL, - est_control, - - NULL, - NULL -}; - -module_info *modules[] = { - (module_info *)&est_driver_module, - (module_info *)&est_device_module, - NULL -}; diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h b/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h deleted file mode 100644 index b51fa9120f..0000000000 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/enhanced_speedstep.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef _EST_H -#define _EST_H - -#include -#include - -#include "frequency.h" - -// Model Specific Register -#define MSR_MISC 0x1a0 -#define MSR_EST_ENABLED (1<<16) - - -struct est_cookie { - // this three variables are not needed yet but helpfull when extend this - // driver to use acpi - device_node *node; - acpi_device_module_info *acpi; - acpi_device acpi_cookie; - - // array of states don't delete it - freq_info* available_states; - uint8 number_states; - - int32 stop_watching; -}; - - -#endif /* _EST_H */ diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.cpp b/src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.cpp deleted file mode 100644 index 0c75901c77..0000000000 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - - -#include "frequency.h" - -#include - -#include -#include -#include - - -void -est_get_id16(uint16* _id16) -{ - *_id16 = x86_read_msr(MSR_GET_FREQ_STATE) & 0xffff; -} - - -status_t -est_set_id16(uint16 id16, bool needCheck) -{ - uint64 msr; - - // Read the current register, mask out the old, set the new id. - msr = x86_read_msr(MSR_GET_FREQ_STATE); - msr = (msr & ~0xffff) | id16; - x86_write_msr(MSR_SET_FREQ_STATE, msr); - - if (needCheck) { - // Wait a short while for the new setting. XXX Is this necessary? - snooze(EST_TRANS_LAT); - - uint16 newID16; - est_get_id16(&newID16); - if (newID16 != id16) - return B_ERROR; - TRACE("EST: set frequency ok, id %i\n", id16); - } - return B_OK; -} - - -freq_info* -est_get_current(freq_info* list) -{ - // Try a few times to get a valid value. Sometimes, if the CPU - // is in the middle of an asynchronous transition (i.e., P4TCC), - // we get a temporary invalid result. - for (uint32 i = 0; i < 5; i++) { - uint16 id16; - est_get_id16(&id16); - - for (freq_info* info = list; info->id != 0; info++) { - if (info->id == id16) - return info; - } - - snooze(100); - } - return NULL; -} - - -status_t -est_get_info(freq_info** _frequencyInfos) -{ - uint64 msr = x86_read_msr(MSR_GET_FREQ_STATE); - status_t status = est_table_info(msr, _frequencyInfos); - if (status != B_OK) { - TRACE("EST: Get frequency table from model specific register\n"); - status = est_msr_info(msr, _frequencyInfos); - } - - if (status != B_OK) { - TRACE("est: CPU supports Enhanced Speedstep, but is not recognized.\n"); - return status; - } - - return B_OK; -} - - -status_t -est_table_info(uint64 msr, freq_info** _frequencyInfos) -{ - // Find a table which matches (vendor, id32). - system_info info; - if (get_system_info(&info) != B_OK) - return B_ERROR; - - ss_cpu_info* cpuInfo; - uint32 id = msr >> 32; - for (cpuInfo = ESTprocs; cpuInfo->id32 != 0; cpuInfo++) { - if (cpuInfo->vendor_id == uint32(info.cpu_type & B_CPU_x86_VENDOR_MASK) - && cpuInfo->id32 == id) - break; - } - if (cpuInfo->id32 == 0) - return B_ERROR; - - // Make sure the current setpoint is valid. - if (est_get_current(cpuInfo->freqtab) == NULL) { - TRACE("current setting not found in table\n"); - return B_ERROR; - } - - *_frequencyInfos = cpuInfo->freqtab; - return B_OK; -} - - -bool -bus_speed_ok(int bus) -{ - switch (bus) { - case 100: - case 133: - case 166: - case 333: - return true; - default: - return false; - } -} - - -/*! Flesh out a simple rate table containing the high and low frequencies - based on the current clock speed and the upper 32 bits of the MSR. -*/ -status_t -est_msr_info(uint64 msr, freq_info** _frequencyInfos) -{ - // Figure out the bus clock. - system_info info; - if (get_system_info(&info) != B_OK) - return B_ERROR; - - int32 freq = info.cpu_clock_speed / 1000000; - uint16 id = msr >> 32; - int32 bus = 0; - if (id >> 8) - bus = freq / (id >> 8); - - TRACE("est: Guessed bus clock (high) of %d MHz\n", int(bus)); - if (!bus_speed_ok(bus)) { - // We may be running on the low frequency. - id = msr >> 48; - if (id >> 8) - bus = freq / (id >> 8); - TRACE("est: Guessed bus clock (low) of %d MHz\n", int(bus)); - if (!bus_speed_ok(bus)) - return B_ERROR; - - // Calculate high frequency. - id = msr >> 32; - freq = ((id >> 8) & 0xff) * bus; - } - - // Fill out a new freq table containing just the high and low freqs. - freq_info* frequencyInfo = (freq_info*)malloc(sizeof(freq_info) * 3); - if (frequencyInfo == NULL) - return B_NO_MEMORY; - - memset(frequencyInfo, 0, sizeof(freq_info) * 3); - - // First, the high frequency. - int32 volts = id & 0xff; - if (volts != 0) { - volts <<= 4; - volts += 700; - } - frequencyInfo[0].frequency = freq; - frequencyInfo[0].volts = volts; - frequencyInfo[0].id = id; - frequencyInfo[0].power = CPUFREQ_VAL_UNKNOWN; - TRACE("Guessed high setting of %d MHz @ %d Mv\n", int(freq), int(volts)); - - // Second, the low frequency. - id = msr >> 48; - freq = ((id >> 8) & 0xff) * bus; - volts = id & 0xff; - if (volts != 0) { - volts <<= 4; - volts += 700; - } - frequencyInfo[1].frequency = freq; - frequencyInfo[1].volts = volts; - frequencyInfo[1].id = id; - frequencyInfo[1].power = CPUFREQ_VAL_UNKNOWN; - TRACE("Guessed low setting of %d MHz @ %d Mv\n", int(freq), int(volts)); - - // Table is already terminated due to M_ZERO. - *_frequencyInfos = frequencyInfo; - return B_OK; -} diff --git a/src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.h b/src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.h deleted file mode 100644 index 454bc6e2bb..0000000000 --- a/src/add-ons/kernel/drivers/power/enhanced_speedstep/frequency.h +++ /dev/null @@ -1,852 +0,0 @@ -#ifndef FREQ_TABLE_H -#define FREQ_TABLE_H - -#include -#include -#include - -#include "device/power_managment.h" - -#define TRACE_EST -#ifdef TRACE_EST -# define TRACE(x...) dprintf(x) -#else -# define TRACE(x...) -#endif - -// this code is taken from freeBSD - -#define MSR_GET_FREQ_STATE 0x198 -#define MSR_SET_FREQ_STATE 0x199 - -/* Estimate in microseconds of latency for performing a transition. */ -#define EST_TRANS_LAT 1000 - -/* Identifying characteristics of a processor and supported frequencies. */ -typedef struct { - const uint32 vendor_id; - uint32 id32; - freq_info *freqtab; -} ss_cpu_info; - - -status_t est_get_info(freq_info **freqsInfo); -status_t est_table_info(uint64 msr, freq_info **freqs); -status_t est_msr_info(uint64 msr, freq_info **freqs); - -freq_info* est_get_current(freq_info *freq_list); -void est_get_id16(uint16 *id16_p); -status_t est_set_id16(uint16 id16, bool need_check = false); - -/* Convert MHz and mV into IDs for passing to the MSR. */ -#define ID16(MHz, mV, bus_clk) \ - (((MHz / bus_clk) << 8) | ((mV ? mV - 700 : 0) >> 4)) -#define ID32(MHz_hi, mV_hi, MHz_lo, mV_lo, bus_clk) \ - ((ID16(MHz_lo, mV_lo, bus_clk) << 16) | (ID16(MHz_hi, mV_hi, bus_clk))) - -/* Format for storing IDs in our table. */ -#define CPUFREQ_VAL_UNKNOWN (-1) -#define CPU_VENDOR_INTEL B_CPU_INTEL_x86 -#define CPU_VENDOR_CENTAUR B_CPU_VIA_IDT_x86 - -#define FREQ_INFO_PWR(MHz, mV, bus_clk, mW) \ - { MHz, mV, ID16(MHz, mV, bus_clk), mW } -#define FREQ_INFO(MHz, mV, bus_clk) \ - FREQ_INFO_PWR(MHz, mV, bus_clk, CPUFREQ_VAL_UNKNOWN) -#define INTEL(tab, zhi, vhi, zlo, vlo, bus_clk) \ - { CPU_VENDOR_INTEL, ID32(zhi, vhi, zlo, vlo, bus_clk), tab } -#define CENTAUR(tab, zhi, vhi, zlo, vlo, bus_clk) \ - { CPU_VENDOR_CENTAUR, ID32(zhi, vhi, zlo, vlo, bus_clk), tab } - -/* Default bus clock value for Centrino processors. */ -#define INTEL_BUS_CLK 100 - -/* - * Frequency (MHz) and voltage (mV) settings. Data from the - * Intel Pentium M Processor Datasheet (Order Number 252612), Table 5. - * - * Dothan processors have multiple VID#s with different settings for - * each VID#. Since we can't uniquely identify this info - * without undisclosed methods from Intel, we can't support newer - * processors with this table method. If ACPI Px states are supported, - * we get info from them. - */ -static freq_info PM17_130[] = { - /* 130nm 1.70GHz Pentium M */ - FREQ_INFO(1700, 1484, INTEL_BUS_CLK), - FREQ_INFO(1400, 1308, INTEL_BUS_CLK), - FREQ_INFO(1200, 1228, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1004, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM16_130[] = { - /* 130nm 1.60GHz Pentium M */ - FREQ_INFO(1600, 1484, INTEL_BUS_CLK), - FREQ_INFO(1400, 1420, INTEL_BUS_CLK), - FREQ_INFO(1200, 1276, INTEL_BUS_CLK), - FREQ_INFO(1000, 1164, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM15_130[] = { - /* 130nm 1.50GHz Pentium M */ - FREQ_INFO(1500, 1484, INTEL_BUS_CLK), - FREQ_INFO(1400, 1452, INTEL_BUS_CLK), - FREQ_INFO(1200, 1356, INTEL_BUS_CLK), - FREQ_INFO(1000, 1228, INTEL_BUS_CLK), - FREQ_INFO( 800, 1116, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM14_130[] = { - /* 130nm 1.40GHz Pentium M */ - FREQ_INFO(1400, 1484, INTEL_BUS_CLK), - FREQ_INFO(1200, 1436, INTEL_BUS_CLK), - FREQ_INFO(1000, 1308, INTEL_BUS_CLK), - FREQ_INFO( 800, 1180, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM13_130[] = { - /* 130nm 1.30GHz Pentium M */ - FREQ_INFO(1300, 1388, INTEL_BUS_CLK), - FREQ_INFO(1200, 1356, INTEL_BUS_CLK), - FREQ_INFO(1000, 1292, INTEL_BUS_CLK), - FREQ_INFO( 800, 1260, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM13_LV_130[] = { - /* 130nm 1.30GHz Low Voltage Pentium M */ - FREQ_INFO(1300, 1180, INTEL_BUS_CLK), - FREQ_INFO(1200, 1164, INTEL_BUS_CLK), - FREQ_INFO(1100, 1100, INTEL_BUS_CLK), - FREQ_INFO(1000, 1020, INTEL_BUS_CLK), - FREQ_INFO( 900, 1004, INTEL_BUS_CLK), - FREQ_INFO( 800, 988, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM12_LV_130[] = { - /* 130 nm 1.20GHz Low Voltage Pentium M */ - FREQ_INFO(1200, 1180, INTEL_BUS_CLK), - FREQ_INFO(1100, 1164, INTEL_BUS_CLK), - FREQ_INFO(1000, 1100, INTEL_BUS_CLK), - FREQ_INFO( 900, 1020, INTEL_BUS_CLK), - FREQ_INFO( 800, 1004, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM11_LV_130[] = { - /* 130 nm 1.10GHz Low Voltage Pentium M */ - FREQ_INFO(1100, 1180, INTEL_BUS_CLK), - FREQ_INFO(1000, 1164, INTEL_BUS_CLK), - FREQ_INFO( 900, 1100, INTEL_BUS_CLK), - FREQ_INFO( 800, 1020, INTEL_BUS_CLK), - FREQ_INFO( 600, 956, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM11_ULV_130[] = { - /* 130 nm 1.10GHz Ultra Low Voltage Pentium M */ - FREQ_INFO(1100, 1004, INTEL_BUS_CLK), - FREQ_INFO(1000, 988, INTEL_BUS_CLK), - FREQ_INFO( 900, 972, INTEL_BUS_CLK), - FREQ_INFO( 800, 956, INTEL_BUS_CLK), - FREQ_INFO( 600, 844, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM10_ULV_130[] = { - /* 130 nm 1.00GHz Ultra Low Voltage Pentium M */ - FREQ_INFO(1000, 1004, INTEL_BUS_CLK), - FREQ_INFO( 900, 988, INTEL_BUS_CLK), - FREQ_INFO( 800, 972, INTEL_BUS_CLK), - FREQ_INFO( 600, 844, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; - -/* - * Data from "Intel Pentium M Processor on 90nm Process with - * 2-MB L2 Cache Datasheet", Order Number 302189, Table 5. - */ -static freq_info PM_765A_90[] = { - /* 90 nm 2.10GHz Pentium M, VID #A */ - FREQ_INFO(2100, 1340, INTEL_BUS_CLK), - FREQ_INFO(1800, 1276, INTEL_BUS_CLK), - FREQ_INFO(1600, 1228, INTEL_BUS_CLK), - FREQ_INFO(1400, 1180, INTEL_BUS_CLK), - FREQ_INFO(1200, 1132, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_765B_90[] = { - /* 90 nm 2.10GHz Pentium M, VID #B */ - FREQ_INFO(2100, 1324, INTEL_BUS_CLK), - FREQ_INFO(1800, 1260, INTEL_BUS_CLK), - FREQ_INFO(1600, 1212, INTEL_BUS_CLK), - FREQ_INFO(1400, 1180, INTEL_BUS_CLK), - FREQ_INFO(1200, 1132, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_765C_90[] = { - /* 90 nm 2.10GHz Pentium M, VID #C */ - FREQ_INFO(2100, 1308, INTEL_BUS_CLK), - FREQ_INFO(1800, 1244, INTEL_BUS_CLK), - FREQ_INFO(1600, 1212, INTEL_BUS_CLK), - FREQ_INFO(1400, 1164, INTEL_BUS_CLK), - FREQ_INFO(1200, 1116, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_765E_90[] = { - /* 90 nm 2.10GHz Pentium M, VID #E */ - FREQ_INFO(2100, 1356, INTEL_BUS_CLK), - FREQ_INFO(1800, 1292, INTEL_BUS_CLK), - FREQ_INFO(1600, 1244, INTEL_BUS_CLK), - FREQ_INFO(1400, 1196, INTEL_BUS_CLK), - FREQ_INFO(1200, 1148, INTEL_BUS_CLK), - FREQ_INFO(1000, 1100, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_755A_90[] = { - /* 90 nm 2.00GHz Pentium M, VID #A */ - FREQ_INFO(2000, 1340, INTEL_BUS_CLK), - FREQ_INFO(1800, 1292, INTEL_BUS_CLK), - FREQ_INFO(1600, 1244, INTEL_BUS_CLK), - FREQ_INFO(1400, 1196, INTEL_BUS_CLK), - FREQ_INFO(1200, 1148, INTEL_BUS_CLK), - FREQ_INFO(1000, 1100, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_755B_90[] = { - /* 90 nm 2.00GHz Pentium M, VID #B */ - FREQ_INFO(2000, 1324, INTEL_BUS_CLK), - FREQ_INFO(1800, 1276, INTEL_BUS_CLK), - FREQ_INFO(1600, 1228, INTEL_BUS_CLK), - FREQ_INFO(1400, 1180, INTEL_BUS_CLK), - FREQ_INFO(1200, 1132, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_755C_90[] = { - /* 90 nm 2.00GHz Pentium M, VID #C */ - FREQ_INFO(2000, 1308, INTEL_BUS_CLK), - FREQ_INFO(1800, 1276, INTEL_BUS_CLK), - FREQ_INFO(1600, 1228, INTEL_BUS_CLK), - FREQ_INFO(1400, 1180, INTEL_BUS_CLK), - FREQ_INFO(1200, 1132, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_755D_90[] = { - /* 90 nm 2.00GHz Pentium M, VID #D */ - FREQ_INFO(2000, 1276, INTEL_BUS_CLK), - FREQ_INFO(1800, 1244, INTEL_BUS_CLK), - FREQ_INFO(1600, 1196, INTEL_BUS_CLK), - FREQ_INFO(1400, 1164, INTEL_BUS_CLK), - FREQ_INFO(1200, 1116, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_745A_90[] = { - /* 90 nm 1.80GHz Pentium M, VID #A */ - FREQ_INFO(1800, 1340, INTEL_BUS_CLK), - FREQ_INFO(1600, 1292, INTEL_BUS_CLK), - FREQ_INFO(1400, 1228, INTEL_BUS_CLK), - FREQ_INFO(1200, 1164, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_745B_90[] = { - /* 90 nm 1.80GHz Pentium M, VID #B */ - FREQ_INFO(1800, 1324, INTEL_BUS_CLK), - FREQ_INFO(1600, 1276, INTEL_BUS_CLK), - FREQ_INFO(1400, 1212, INTEL_BUS_CLK), - FREQ_INFO(1200, 1164, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_745C_90[] = { - /* 90 nm 1.80GHz Pentium M, VID #C */ - FREQ_INFO(1800, 1308, INTEL_BUS_CLK), - FREQ_INFO(1600, 1260, INTEL_BUS_CLK), - FREQ_INFO(1400, 1212, INTEL_BUS_CLK), - FREQ_INFO(1200, 1148, INTEL_BUS_CLK), - FREQ_INFO(1000, 1100, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_745D_90[] = { - /* 90 nm 1.80GHz Pentium M, VID #D */ - FREQ_INFO(1800, 1276, INTEL_BUS_CLK), - FREQ_INFO(1600, 1228, INTEL_BUS_CLK), - FREQ_INFO(1400, 1180, INTEL_BUS_CLK), - FREQ_INFO(1200, 1132, INTEL_BUS_CLK), - FREQ_INFO(1000, 1084, INTEL_BUS_CLK), - FREQ_INFO( 800, 1036, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_735A_90[] = { - /* 90 nm 1.70GHz Pentium M, VID #A */ - FREQ_INFO(1700, 1340, INTEL_BUS_CLK), - FREQ_INFO(1400, 1244, INTEL_BUS_CLK), - FREQ_INFO(1200, 1180, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_735B_90[] = { - /* 90 nm 1.70GHz Pentium M, VID #B */ - FREQ_INFO(1700, 1324, INTEL_BUS_CLK), - FREQ_INFO(1400, 1244, INTEL_BUS_CLK), - FREQ_INFO(1200, 1180, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_735C_90[] = { - /* 90 nm 1.70GHz Pentium M, VID #C */ - FREQ_INFO(1700, 1308, INTEL_BUS_CLK), - FREQ_INFO(1400, 1228, INTEL_BUS_CLK), - FREQ_INFO(1200, 1164, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_735D_90[] = { - /* 90 nm 1.70GHz Pentium M, VID #D */ - FREQ_INFO(1700, 1276, INTEL_BUS_CLK), - FREQ_INFO(1400, 1212, INTEL_BUS_CLK), - FREQ_INFO(1200, 1148, INTEL_BUS_CLK), - FREQ_INFO(1000, 1100, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_725A_90[] = { - /* 90 nm 1.60GHz Pentium M, VID #A */ - FREQ_INFO(1600, 1340, INTEL_BUS_CLK), - FREQ_INFO(1400, 1276, INTEL_BUS_CLK), - FREQ_INFO(1200, 1212, INTEL_BUS_CLK), - FREQ_INFO(1000, 1132, INTEL_BUS_CLK), - FREQ_INFO( 800, 1068, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_725B_90[] = { - /* 90 nm 1.60GHz Pentium M, VID #B */ - FREQ_INFO(1600, 1324, INTEL_BUS_CLK), - FREQ_INFO(1400, 1260, INTEL_BUS_CLK), - FREQ_INFO(1200, 1196, INTEL_BUS_CLK), - FREQ_INFO(1000, 1132, INTEL_BUS_CLK), - FREQ_INFO( 800, 1068, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_725C_90[] = { - /* 90 nm 1.60GHz Pentium M, VID #C */ - FREQ_INFO(1600, 1308, INTEL_BUS_CLK), - FREQ_INFO(1400, 1244, INTEL_BUS_CLK), - FREQ_INFO(1200, 1180, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_725D_90[] = { - /* 90 nm 1.60GHz Pentium M, VID #D */ - FREQ_INFO(1600, 1276, INTEL_BUS_CLK), - FREQ_INFO(1400, 1228, INTEL_BUS_CLK), - FREQ_INFO(1200, 1164, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_715A_90[] = { - /* 90 nm 1.50GHz Pentium M, VID #A */ - FREQ_INFO(1500, 1340, INTEL_BUS_CLK), - FREQ_INFO(1200, 1228, INTEL_BUS_CLK), - FREQ_INFO(1000, 1148, INTEL_BUS_CLK), - FREQ_INFO( 800, 1068, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_715B_90[] = { - /* 90 nm 1.50GHz Pentium M, VID #B */ - FREQ_INFO(1500, 1324, INTEL_BUS_CLK), - FREQ_INFO(1200, 1212, INTEL_BUS_CLK), - FREQ_INFO(1000, 1148, INTEL_BUS_CLK), - FREQ_INFO( 800, 1068, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_715C_90[] = { - /* 90 nm 1.50GHz Pentium M, VID #C */ - FREQ_INFO(1500, 1308, INTEL_BUS_CLK), - FREQ_INFO(1200, 1212, INTEL_BUS_CLK), - FREQ_INFO(1000, 1132, INTEL_BUS_CLK), - FREQ_INFO( 800, 1068, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_715D_90[] = { - /* 90 nm 1.50GHz Pentium M, VID #D */ - FREQ_INFO(1500, 1276, INTEL_BUS_CLK), - FREQ_INFO(1200, 1180, INTEL_BUS_CLK), - FREQ_INFO(1000, 1116, INTEL_BUS_CLK), - FREQ_INFO( 800, 1052, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_778_90[] = { - /* 90 nm 1.60GHz Low Voltage Pentium M */ - FREQ_INFO(1600, 1116, INTEL_BUS_CLK), - FREQ_INFO(1500, 1116, INTEL_BUS_CLK), - FREQ_INFO(1400, 1100, INTEL_BUS_CLK), - FREQ_INFO(1300, 1084, INTEL_BUS_CLK), - FREQ_INFO(1200, 1068, INTEL_BUS_CLK), - FREQ_INFO(1100, 1052, INTEL_BUS_CLK), - FREQ_INFO(1000, 1052, INTEL_BUS_CLK), - FREQ_INFO( 900, 1036, INTEL_BUS_CLK), - FREQ_INFO( 800, 1020, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_758_90[] = { - /* 90 nm 1.50GHz Low Voltage Pentium M */ - FREQ_INFO(1500, 1116, INTEL_BUS_CLK), - FREQ_INFO(1400, 1116, INTEL_BUS_CLK), - FREQ_INFO(1300, 1100, INTEL_BUS_CLK), - FREQ_INFO(1200, 1084, INTEL_BUS_CLK), - FREQ_INFO(1100, 1068, INTEL_BUS_CLK), - FREQ_INFO(1000, 1052, INTEL_BUS_CLK), - FREQ_INFO( 900, 1036, INTEL_BUS_CLK), - FREQ_INFO( 800, 1020, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_738_90[] = { - /* 90 nm 1.40GHz Low Voltage Pentium M */ - FREQ_INFO(1400, 1116, INTEL_BUS_CLK), - FREQ_INFO(1300, 1116, INTEL_BUS_CLK), - FREQ_INFO(1200, 1100, INTEL_BUS_CLK), - FREQ_INFO(1100, 1068, INTEL_BUS_CLK), - FREQ_INFO(1000, 1052, INTEL_BUS_CLK), - FREQ_INFO( 900, 1036, INTEL_BUS_CLK), - FREQ_INFO( 800, 1020, INTEL_BUS_CLK), - FREQ_INFO( 600, 988, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_773G_90[] = { - /* 90 nm 1.30GHz Ultra Low Voltage Pentium M, VID #G */ - FREQ_INFO(1300, 956, INTEL_BUS_CLK), - FREQ_INFO(1200, 940, INTEL_BUS_CLK), - FREQ_INFO(1100, 924, INTEL_BUS_CLK), - FREQ_INFO(1000, 908, INTEL_BUS_CLK), - FREQ_INFO( 900, 876, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_773H_90[] = { - /* 90 nm 1.30GHz Ultra Low Voltage Pentium M, VID #H */ - FREQ_INFO(1300, 940, INTEL_BUS_CLK), - FREQ_INFO(1200, 924, INTEL_BUS_CLK), - FREQ_INFO(1100, 908, INTEL_BUS_CLK), - FREQ_INFO(1000, 892, INTEL_BUS_CLK), - FREQ_INFO( 900, 876, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_773I_90[] = { - /* 90 nm 1.30GHz Ultra Low Voltage Pentium M, VID #I */ - FREQ_INFO(1300, 924, INTEL_BUS_CLK), - FREQ_INFO(1200, 908, INTEL_BUS_CLK), - FREQ_INFO(1100, 892, INTEL_BUS_CLK), - FREQ_INFO(1000, 876, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_773J_90[] = { - /* 90 nm 1.30GHz Ultra Low Voltage Pentium M, VID #J */ - FREQ_INFO(1300, 908, INTEL_BUS_CLK), - FREQ_INFO(1200, 908, INTEL_BUS_CLK), - FREQ_INFO(1100, 892, INTEL_BUS_CLK), - FREQ_INFO(1000, 876, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_773K_90[] = { - /* 90 nm 1.30GHz Ultra Low Voltage Pentium M, VID #K */ - FREQ_INFO(1300, 892, INTEL_BUS_CLK), - FREQ_INFO(1200, 892, INTEL_BUS_CLK), - FREQ_INFO(1100, 876, INTEL_BUS_CLK), - FREQ_INFO(1000, 860, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_773L_90[] = { - /* 90 nm 1.30GHz Ultra Low Voltage Pentium M, VID #L */ - FREQ_INFO(1300, 876, INTEL_BUS_CLK), - FREQ_INFO(1200, 876, INTEL_BUS_CLK), - FREQ_INFO(1100, 860, INTEL_BUS_CLK), - FREQ_INFO(1000, 860, INTEL_BUS_CLK), - FREQ_INFO( 900, 844, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_753G_90[] = { - /* 90 nm 1.20GHz Ultra Low Voltage Pentium M, VID #G */ - FREQ_INFO(1200, 956, INTEL_BUS_CLK), - FREQ_INFO(1100, 940, INTEL_BUS_CLK), - FREQ_INFO(1000, 908, INTEL_BUS_CLK), - FREQ_INFO( 900, 892, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_753H_90[] = { - /* 90 nm 1.20GHz Ultra Low Voltage Pentium M, VID #H */ - FREQ_INFO(1200, 940, INTEL_BUS_CLK), - FREQ_INFO(1100, 924, INTEL_BUS_CLK), - FREQ_INFO(1000, 908, INTEL_BUS_CLK), - FREQ_INFO( 900, 876, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_753I_90[] = { - /* 90 nm 1.20GHz Ultra Low Voltage Pentium M, VID #I */ - FREQ_INFO(1200, 924, INTEL_BUS_CLK), - FREQ_INFO(1100, 908, INTEL_BUS_CLK), - FREQ_INFO(1000, 892, INTEL_BUS_CLK), - FREQ_INFO( 900, 876, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_753J_90[] = { - /* 90 nm 1.20GHz Ultra Low Voltage Pentium M, VID #J */ - FREQ_INFO(1200, 908, INTEL_BUS_CLK), - FREQ_INFO(1100, 892, INTEL_BUS_CLK), - FREQ_INFO(1000, 876, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_753K_90[] = { - /* 90 nm 1.20GHz Ultra Low Voltage Pentium M, VID #K */ - FREQ_INFO(1200, 892, INTEL_BUS_CLK), - FREQ_INFO(1100, 892, INTEL_BUS_CLK), - FREQ_INFO(1000, 876, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_753L_90[] = { - /* 90 nm 1.20GHz Ultra Low Voltage Pentium M, VID #L */ - FREQ_INFO(1200, 876, INTEL_BUS_CLK), - FREQ_INFO(1100, 876, INTEL_BUS_CLK), - FREQ_INFO(1000, 860, INTEL_BUS_CLK), - FREQ_INFO( 900, 844, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; - -static freq_info PM_733JG_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M, VID #G */ - FREQ_INFO(1100, 956, INTEL_BUS_CLK), - FREQ_INFO(1000, 940, INTEL_BUS_CLK), - FREQ_INFO( 900, 908, INTEL_BUS_CLK), - FREQ_INFO( 800, 876, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_733JH_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M, VID #H */ - FREQ_INFO(1100, 940, INTEL_BUS_CLK), - FREQ_INFO(1000, 924, INTEL_BUS_CLK), - FREQ_INFO( 900, 892, INTEL_BUS_CLK), - FREQ_INFO( 800, 876, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_733JI_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M, VID #I */ - FREQ_INFO(1100, 924, INTEL_BUS_CLK), - FREQ_INFO(1000, 908, INTEL_BUS_CLK), - FREQ_INFO( 900, 892, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_733JJ_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M, VID #J */ - FREQ_INFO(1100, 908, INTEL_BUS_CLK), - FREQ_INFO(1000, 892, INTEL_BUS_CLK), - FREQ_INFO( 900, 876, INTEL_BUS_CLK), - FREQ_INFO( 800, 860, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_733JK_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M, VID #K */ - FREQ_INFO(1100, 892, INTEL_BUS_CLK), - FREQ_INFO(1000, 876, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_733JL_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M, VID #L */ - FREQ_INFO(1100, 876, INTEL_BUS_CLK), - FREQ_INFO(1000, 876, INTEL_BUS_CLK), - FREQ_INFO( 900, 860, INTEL_BUS_CLK), - FREQ_INFO( 800, 844, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), -}; -static freq_info PM_733_90[] = { - /* 90 nm 1.10GHz Ultra Low Voltage Pentium M */ - FREQ_INFO(1100, 940, INTEL_BUS_CLK), - FREQ_INFO(1000, 924, INTEL_BUS_CLK), - FREQ_INFO( 900, 892, INTEL_BUS_CLK), - FREQ_INFO( 800, 876, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; -static freq_info PM_723_90[] = { - /* 90 nm 1.00GHz Ultra Low Voltage Pentium M */ - FREQ_INFO(1000, 940, INTEL_BUS_CLK), - FREQ_INFO( 900, 908, INTEL_BUS_CLK), - FREQ_INFO( 800, 876, INTEL_BUS_CLK), - FREQ_INFO( 600, 812, INTEL_BUS_CLK), - FREQ_INFO( 0, 0, 1), -}; - -/* - * VIA C7-M 500 MHz FSB, 400 MHz FSB, and ULV variants. - * Data from the "VIA C7-M Processor BIOS Writer's Guide (v2.17)" datasheet. - */ -static freq_info C7M_795[] = { - /* 2.00GHz Centaur C7-M 533 Mhz FSB */ - FREQ_INFO_PWR(2000, 1148, 133, 20000), - FREQ_INFO_PWR(1867, 1132, 133, 18000), - FREQ_INFO_PWR(1600, 1100, 133, 15000), - FREQ_INFO_PWR(1467, 1052, 133, 13000), - FREQ_INFO_PWR(1200, 1004, 133, 10000), - FREQ_INFO_PWR( 800, 844, 133, 7000), - FREQ_INFO_PWR( 667, 844, 133, 6000), - FREQ_INFO_PWR( 533, 844, 133, 5000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_785[] = { - /* 1.80GHz Centaur C7-M 533 Mhz FSB */ - FREQ_INFO_PWR(1867, 1148, 133, 18000), - FREQ_INFO_PWR(1600, 1100, 133, 15000), - FREQ_INFO_PWR(1467, 1052, 133, 13000), - FREQ_INFO_PWR(1200, 1004, 133, 10000), - FREQ_INFO_PWR( 800, 844, 133, 7000), - FREQ_INFO_PWR( 667, 844, 133, 6000), - FREQ_INFO_PWR( 533, 844, 133, 5000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_765[] = { - /* 1.60GHz Centaur C7-M 533 Mhz FSB */ - FREQ_INFO_PWR(1600, 1084, 133, 15000), - FREQ_INFO_PWR(1467, 1052, 133, 13000), - FREQ_INFO_PWR(1200, 1004, 133, 10000), - FREQ_INFO_PWR( 800, 844, 133, 7000), - FREQ_INFO_PWR( 667, 844, 133, 6000), - FREQ_INFO_PWR( 533, 844, 133, 5000), - FREQ_INFO(0, 0, 1), -}; - -static freq_info C7M_794[] = { - /* 2.00GHz Centaur C7-M 400 Mhz FSB */ - FREQ_INFO_PWR(2000, 1148, 100, 20000), - FREQ_INFO_PWR(1800, 1132, 100, 18000), - FREQ_INFO_PWR(1600, 1100, 100, 15000), - FREQ_INFO_PWR(1400, 1052, 100, 13000), - FREQ_INFO_PWR(1000, 1004, 100, 10000), - FREQ_INFO_PWR( 800, 844, 100, 7000), - FREQ_INFO_PWR( 600, 844, 100, 6000), - FREQ_INFO_PWR( 400, 844, 100, 5000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_784[] = { - /* 1.80GHz Centaur C7-M 400 Mhz FSB */ - FREQ_INFO_PWR(1800, 1148, 100, 18000), - FREQ_INFO_PWR(1600, 1100, 100, 15000), - FREQ_INFO_PWR(1400, 1052, 100, 13000), - FREQ_INFO_PWR(1000, 1004, 100, 10000), - FREQ_INFO_PWR( 800, 844, 100, 7000), - FREQ_INFO_PWR( 600, 844, 100, 6000), - FREQ_INFO_PWR( 400, 844, 100, 5000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_764[] = { - /* 1.60GHz Centaur C7-M 400 Mhz FSB */ - FREQ_INFO_PWR(1600, 1084, 100, 15000), - FREQ_INFO_PWR(1400, 1052, 100, 13000), - FREQ_INFO_PWR(1000, 1004, 100, 10000), - FREQ_INFO_PWR( 800, 844, 100, 7000), - FREQ_INFO_PWR( 600, 844, 100, 6000), - FREQ_INFO_PWR( 400, 844, 100, 5000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_754[] = { - /* 1.50GHz Centaur C7-M 400 Mhz FSB */ - FREQ_INFO_PWR(1500, 1004, 100, 12000), - FREQ_INFO_PWR(1400, 988, 100, 11000), - FREQ_INFO_PWR(1000, 940, 100, 9000), - FREQ_INFO_PWR( 800, 844, 100, 7000), - FREQ_INFO_PWR( 600, 844, 100, 6000), - FREQ_INFO_PWR( 400, 844, 100, 5000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_771[] = { - /* 1.20GHz Centaur C7-M 400 Mhz FSB */ - FREQ_INFO_PWR(1200, 860, 100, 7000), - FREQ_INFO_PWR(1000, 860, 100, 6000), - FREQ_INFO_PWR( 800, 844, 100, 5500), - FREQ_INFO_PWR( 600, 844, 100, 5000), - FREQ_INFO_PWR( 400, 844, 100, 4000), - FREQ_INFO(0, 0, 1), -}; - -static freq_info C7M_775_ULV[] = { - /* 1.50GHz Centaur C7-M ULV */ - FREQ_INFO_PWR(1500, 956, 100, 7500), - FREQ_INFO_PWR(1400, 940, 100, 6000), - FREQ_INFO_PWR(1000, 860, 100, 5000), - FREQ_INFO_PWR( 800, 828, 100, 2800), - FREQ_INFO_PWR( 600, 796, 100, 2500), - FREQ_INFO_PWR( 400, 796, 100, 2000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_772_ULV[] = { - /* 1.20GHz Centaur C7-M ULV */ - FREQ_INFO_PWR(1200, 844, 100, 5000), - FREQ_INFO_PWR(1000, 844, 100, 4000), - FREQ_INFO_PWR( 800, 828, 100, 2800), - FREQ_INFO_PWR( 600, 796, 100, 2500), - FREQ_INFO_PWR( 400, 796, 100, 2000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_779_ULV[] = { - /* 1.00GHz Centaur C7-M ULV */ - FREQ_INFO_PWR(1000, 796, 100, 3500), - FREQ_INFO_PWR( 800, 796, 100, 2800), - FREQ_INFO_PWR( 600, 796, 100, 2500), - FREQ_INFO_PWR( 400, 796, 100, 2000), - FREQ_INFO(0, 0, 1), -}; -static freq_info C7M_770_ULV[] = { - /* 1.00GHz Centaur C7-M ULV */ - FREQ_INFO_PWR(1000, 844, 100, 5000), - FREQ_INFO_PWR( 800, 796, 100, 2800), - FREQ_INFO_PWR( 600, 796, 100, 2500), - FREQ_INFO_PWR( 400, 796, 100, 2000), - FREQ_INFO(0, 0, 1), -}; - -static ss_cpu_info ESTprocs[] = { - INTEL(PM17_130, 1700, 1484, 600, 956, INTEL_BUS_CLK), - INTEL(PM16_130, 1600, 1484, 600, 956, INTEL_BUS_CLK), - INTEL(PM15_130, 1500, 1484, 600, 956, INTEL_BUS_CLK), - INTEL(PM14_130, 1400, 1484, 600, 956, INTEL_BUS_CLK), - INTEL(PM13_130, 1300, 1388, 600, 956, INTEL_BUS_CLK), - INTEL(PM13_LV_130, 1300, 1180, 600, 956, INTEL_BUS_CLK), - INTEL(PM12_LV_130, 1200, 1180, 600, 956, INTEL_BUS_CLK), - INTEL(PM11_LV_130, 1100, 1180, 600, 956, INTEL_BUS_CLK), - INTEL(PM11_ULV_130, 1100, 1004, 600, 844, INTEL_BUS_CLK), - INTEL(PM10_ULV_130, 1000, 1004, 600, 844, INTEL_BUS_CLK), - INTEL(PM_765A_90, 2100, 1340, 600, 988, INTEL_BUS_CLK), - INTEL(PM_765B_90, 2100, 1324, 600, 988, INTEL_BUS_CLK), - INTEL(PM_765C_90, 2100, 1308, 600, 988, INTEL_BUS_CLK), - INTEL(PM_765E_90, 2100, 1356, 600, 988, INTEL_BUS_CLK), - INTEL(PM_755A_90, 2000, 1340, 600, 988, INTEL_BUS_CLK), - INTEL(PM_755B_90, 2000, 1324, 600, 988, INTEL_BUS_CLK), - INTEL(PM_755C_90, 2000, 1308, 600, 988, INTEL_BUS_CLK), - INTEL(PM_755D_90, 2000, 1276, 600, 988, INTEL_BUS_CLK), - INTEL(PM_745A_90, 1800, 1340, 600, 988, INTEL_BUS_CLK), - INTEL(PM_745B_90, 1800, 1324, 600, 988, INTEL_BUS_CLK), - INTEL(PM_745C_90, 1800, 1308, 600, 988, INTEL_BUS_CLK), - INTEL(PM_745D_90, 1800, 1276, 600, 988, INTEL_BUS_CLK), - INTEL(PM_735A_90, 1700, 1340, 600, 988, INTEL_BUS_CLK), - INTEL(PM_735B_90, 1700, 1324, 600, 988, INTEL_BUS_CLK), - INTEL(PM_735C_90, 1700, 1308, 600, 988, INTEL_BUS_CLK), - INTEL(PM_735D_90, 1700, 1276, 600, 988, INTEL_BUS_CLK), - INTEL(PM_725A_90, 1600, 1340, 600, 988, INTEL_BUS_CLK), - INTEL(PM_725B_90, 1600, 1324, 600, 988, INTEL_BUS_CLK), - INTEL(PM_725C_90, 1600, 1308, 600, 988, INTEL_BUS_CLK), - INTEL(PM_725D_90, 1600, 1276, 600, 988, INTEL_BUS_CLK), - INTEL(PM_715A_90, 1500, 1340, 600, 988, INTEL_BUS_CLK), - INTEL(PM_715B_90, 1500, 1324, 600, 988, INTEL_BUS_CLK), - INTEL(PM_715C_90, 1500, 1308, 600, 988, INTEL_BUS_CLK), - INTEL(PM_715D_90, 1500, 1276, 600, 988, INTEL_BUS_CLK), - INTEL(PM_778_90, 1600, 1116, 600, 988, INTEL_BUS_CLK), - INTEL(PM_758_90, 1500, 1116, 600, 988, INTEL_BUS_CLK), - INTEL(PM_738_90, 1400, 1116, 600, 988, INTEL_BUS_CLK), - INTEL(PM_773G_90, 1300, 956, 600, 812, INTEL_BUS_CLK), - INTEL(PM_773H_90, 1300, 940, 600, 812, INTEL_BUS_CLK), - INTEL(PM_773I_90, 1300, 924, 600, 812, INTEL_BUS_CLK), - INTEL(PM_773J_90, 1300, 908, 600, 812, INTEL_BUS_CLK), - INTEL(PM_773K_90, 1300, 892, 600, 812, INTEL_BUS_CLK), - INTEL(PM_773L_90, 1300, 876, 600, 812, INTEL_BUS_CLK), - INTEL(PM_753G_90, 1200, 956, 600, 812, INTEL_BUS_CLK), - INTEL(PM_753H_90, 1200, 940, 600, 812, INTEL_BUS_CLK), - INTEL(PM_753I_90, 1200, 924, 600, 812, INTEL_BUS_CLK), - INTEL(PM_753J_90, 1200, 908, 600, 812, INTEL_BUS_CLK), - INTEL(PM_753K_90, 1200, 892, 600, 812, INTEL_BUS_CLK), - INTEL(PM_753L_90, 1200, 876, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733JG_90, 1100, 956, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733JH_90, 1100, 940, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733JI_90, 1100, 924, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733JJ_90, 1100, 908, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733JK_90, 1100, 892, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733JL_90, 1100, 876, 600, 812, INTEL_BUS_CLK), - INTEL(PM_733_90, 1100, 940, 600, 812, INTEL_BUS_CLK), - INTEL(PM_723_90, 1000, 940, 600, 812, INTEL_BUS_CLK), - - CENTAUR(C7M_795, 2000, 1148, 533, 844, 133), - CENTAUR(C7M_794, 2000, 1148, 400, 844, 100), - CENTAUR(C7M_785, 1867, 1148, 533, 844, 133), - CENTAUR(C7M_784, 1800, 1148, 400, 844, 100), - CENTAUR(C7M_765, 1600, 1084, 533, 844, 133), - CENTAUR(C7M_764, 1600, 1084, 400, 844, 100), - CENTAUR(C7M_754, 1500, 1004, 400, 844, 100), - CENTAUR(C7M_775_ULV, 1500, 956, 400, 796, 100), - CENTAUR(C7M_771, 1200, 860, 400, 844, 100), - CENTAUR(C7M_772_ULV, 1200, 844, 400, 796, 100), - CENTAUR(C7M_779_ULV, 1000, 796, 400, 796, 100), - CENTAUR(C7M_770_ULV, 1000, 844, 400, 796, 100), - { 0, 0, NULL }, -}; - -#endif - From 208f76e578adb0bd495e22ed84d35c67c0ff86de Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 22 Nov 2013 05:24:44 +0100 Subject: [PATCH 139/273] preferences: Remove CPUFrequency We can't reliably set contemporary processors to an arbitrary frequency. There are dependencies between cores and thechnologies like Turbo Boost which may make actual frequency significantly different than the requested. Moreover, it is the scheduler job to decide how much CPU performance is needed and user shouldn't interfere with that. --- build/jam/images/HaikuImage | 2 +- build/jam/images/HaikuImageBootstrap | 2 +- .../preferences/cpufrequency/be.catkeys | 21 - .../preferences/cpufrequency/bg.catkeys | 18 - .../preferences/cpufrequency/cs.catkeys | 19 - .../preferences/cpufrequency/da.catkeys | 19 - .../preferences/cpufrequency/de.catkeys | 21 - .../preferences/cpufrequency/el.catkeys | 20 - .../preferences/cpufrequency/eo.catkeys | 19 - .../preferences/cpufrequency/fi.catkeys | 21 - .../preferences/cpufrequency/fr.catkeys | 21 - .../preferences/cpufrequency/hi.catkeys | 20 - .../preferences/cpufrequency/hr.catkeys | 12 - .../preferences/cpufrequency/hu.catkeys | 21 - .../preferences/cpufrequency/it.catkeys | 19 - .../preferences/cpufrequency/ja.catkeys | 21 - .../preferences/cpufrequency/ko.catkeys | 19 - .../preferences/cpufrequency/lt.catkeys | 21 - .../preferences/cpufrequency/nb.catkeys | 19 - .../preferences/cpufrequency/nl.catkeys | 21 - .../preferences/cpufrequency/pl.catkeys | 21 - .../preferences/cpufrequency/pt.catkeys | 12 - .../preferences/cpufrequency/pt_BR.catkeys | 21 - .../preferences/cpufrequency/pt_br.catkeys | 8 - .../preferences/cpufrequency/ro.catkeys | 20 - .../preferences/cpufrequency/ru.catkeys | 21 - .../preferences/cpufrequency/sk.catkeys | 21 - .../preferences/cpufrequency/sv.catkeys | 21 - .../preferences/cpufrequency/uk.catkeys | 20 - .../preferences/cpufrequency/zh-Hans.catkeys | 20 - .../preferences/cpufrequency/zh_Hans.catkeys | 21 - src/preferences/Jamfile | 1 - .../cpufrequency/CPUFrequencyView.cpp | 252 ------- .../cpufrequency/CPUFrequencyView.h | 91 --- .../cpufrequency/ColorStepView.cpp | 333 -------- src/preferences/cpufrequency/ColorStepView.h | 82 -- .../cpufrequency/DriverInterface.cpp | 222 ------ .../cpufrequency/DriverInterface.h | 53 -- src/preferences/cpufrequency/Jamfile | 24 - .../cpufrequency/PreferencesWindow.h | 481 ------------ src/preferences/cpufrequency/StatusView.cpp | 712 ------------------ src/preferences/cpufrequency/StatusView.h | 135 ---- .../cpufrequency/cpufrequency.rdef | 45 -- src/preferences/cpufrequency/main.cpp | 39 - 44 files changed, 2 insertions(+), 3030 deletions(-) delete mode 100644 data/catalogs/preferences/cpufrequency/be.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/bg.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/cs.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/da.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/de.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/el.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/eo.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/fi.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/fr.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/hi.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/hr.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/hu.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/it.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/ja.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/ko.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/lt.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/nb.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/nl.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/pl.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/pt.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/pt_BR.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/pt_br.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/ro.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/ru.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/sk.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/sv.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/uk.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/zh-Hans.catkeys delete mode 100644 data/catalogs/preferences/cpufrequency/zh_Hans.catkeys delete mode 100644 src/preferences/cpufrequency/CPUFrequencyView.cpp delete mode 100644 src/preferences/cpufrequency/CPUFrequencyView.h delete mode 100644 src/preferences/cpufrequency/ColorStepView.cpp delete mode 100644 src/preferences/cpufrequency/ColorStepView.h delete mode 100644 src/preferences/cpufrequency/DriverInterface.cpp delete mode 100644 src/preferences/cpufrequency/DriverInterface.h delete mode 100644 src/preferences/cpufrequency/Jamfile delete mode 100644 src/preferences/cpufrequency/PreferencesWindow.h delete mode 100644 src/preferences/cpufrequency/StatusView.cpp delete mode 100644 src/preferences/cpufrequency/StatusView.h delete mode 100644 src/preferences/cpufrequency/cpufrequency.rdef delete mode 100644 src/preferences/cpufrequency/main.cpp diff --git a/build/jam/images/HaikuImage b/build/jam/images/HaikuImage index df8b0e9ecf..83d650bf5f 100644 --- a/build/jam/images/HaikuImage +++ b/build/jam/images/HaikuImage @@ -53,7 +53,7 @@ SYSTEM_APPS = [ FFilterByBuildFeatures StyledEdit Terminal TextSearch TV WebWatch Workspaces ] ; SYSTEM_PREFERENCES = [ FFilterByBuildFeatures - Appearance Backgrounds CPUFrequency DataTranslations + Appearance Backgrounds DataTranslations Deskbar E-mail FileTypes Keyboard Keymap Locale Media Mouse Network Notifications Printers Screen ScreenSaver Shortcuts Sounds Time Touchpad Tracker VirtualMemory diff --git a/build/jam/images/HaikuImageBootstrap b/build/jam/images/HaikuImageBootstrap index ebf7d1496a..15a59032a4 100644 --- a/build/jam/images/HaikuImageBootstrap +++ b/build/jam/images/HaikuImageBootstrap @@ -53,7 +53,7 @@ SYSTEM_APPS = [ FFilterByBuildFeatures StyledEdit Terminal TextSearch Workspaces ] ; SYSTEM_PREFERENCES = [ FFilterByBuildFeatures - Appearance Backgrounds CPUFrequency + Appearance Backgrounds Deskbar FileTypes Keyboard Keymap Locale Mouse Network Notifications Screen ScreenSaver Shortcuts Time Touchpad Tracker VirtualMemory diff --git a/data/catalogs/preferences/cpufrequency/be.catkeys b/data/catalogs/preferences/cpufrequency/be.catkeys deleted file mode 100644 index 3c09cb3f55..0000000000 --- a/data/catalogs/preferences/cpufrequency/be.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 belarusian x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Так -CPU frequency status view CPU Frequency View Паказнік стану частаты працэсара -OK Status view ОК -Set state Status view Пазначыць стан -Defaults Pref Window Прадвызначаныя -Install replicant into Deskbar CPU Frequency View Усталяваць рэплікант у Deskbar -Quit Status view Выйсце -High performance Status view Вышэйшая прадукцыйнасць -Stepping policy CPU Frequency View Палітыка пакрокавай рэгуляцыі -Dynamic performance Status view Дынамічная прадукцыйнасць -Step up by CPU usage Color Step View Актывізаваць у залежнасьці ад занятасці працэсара -Dynamic stepping CPU Frequency View Дынамічны рэжым пакрокавай рэгуляцыі -Stepping policy: CPU Frequency View Палітыка пакрокавай рэгуляцыі: -Integration time [ms]: CPU Frequency View Квант інтэграцыі [ms]: -Open Speedstep preferences… Status view Паказаць наладкі Speedstep… -Revert Pref Window Вярнуць -Low energy Status view Экономия энергии -Launching the CPU frequency preflet failed.\n\nError: Status view Немагчыма запусціць канфігурацыйную праграму!\n\nКод памылкі: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPU Frequency\n\tбыў распрацаваны Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Частата Працэсара diff --git a/data/catalogs/preferences/cpufrequency/bg.catkeys b/data/catalogs/preferences/cpufrequency/bg.catkeys deleted file mode 100644 index 763d7c4ea3..0000000000 --- a/data/catalogs/preferences/cpufrequency/bg.catkeys +++ /dev/null @@ -1,18 +0,0 @@ -1 bulgarian x-vnd.Haiku-CPUFrequencyPref 1018039204 -Ok Status view Ок -CPU frequency status view CPU Frequency View Статус на CPU честотата -Set state Status view Изберете състояние -Defaults Pref Window По подразбиране -Install replicant into Deskbar CPU Frequency View Инсталирай репликант в Deskbar -High performance Status view Висока производителност -Stepping policy CPU Frequency View Постъпково изменение на частотата на CPU -Dynamic performance Status view Динамична производителност -Step up by CPU usage Color Step View Стъпка на производителност на CPU: -Dynamic stepping CPU Frequency View Динамично изменение -Stepping policy: CPU Frequency View Постъпково изменение на частотата на CPU: -Integration time [ms]: CPU Frequency View Време на интеграция [мс]: -Open Speedstep preferences… Status view Отвори постъпковите настройки -Revert Pref Window Върни -Low energy Status view Икономия на енергия -Launching the CPU frequency preflet failed.\n\nError: Status view Старта на CPU Frequency не се състоя.\n\nГрешка: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tе разработен от Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n diff --git a/data/catalogs/preferences/cpufrequency/cs.catkeys b/data/catalogs/preferences/cpufrequency/cs.catkeys deleted file mode 100644 index 647de76f32..0000000000 --- a/data/catalogs/preferences/cpufrequency/cs.catkeys +++ /dev/null @@ -1,19 +0,0 @@ -1 czech x-vnd.Haiku-CPUFrequencyPref 3974849433 -CPU frequency status view CPU Frequency View Zobrazení statusu frekvence CPU -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tnapsal Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -Defaults Pref Window Původní -Dynamic performance Status view Dynamický výkon -Dynamic stepping CPU Frequency View Dynamické krokování -High performance Status view Vysoký výkon -Install replicant into Deskbar CPU Frequency View Instaloval replikant do Deskbaru -Integration time [ms]: CPU Frequency View Integrační čas [ms]: -Launching the CPU frequency preflet failed.\n\nError: Status view Nepodařilo se otevřít nastavení frekvence CPU.\n\nChyba: -Low energy Status view Nízká spotřeba -Ok Status view Ok -Open Speedstep preferences… Status view Otevřít nastavení SpeedStep... -Quit Status view Odejít -Revert Pref Window Obnovit -Set state Status view Nastavit stav -Step up by CPU usage Color Step View Krokovat podle využití CPU -Stepping policy CPU Frequency View Krokovací politika -Stepping policy: CPU Frequency View Krokovací politika: diff --git a/data/catalogs/preferences/cpufrequency/da.catkeys b/data/catalogs/preferences/cpufrequency/da.catkeys deleted file mode 100644 index 228fa1028a..0000000000 --- a/data/catalogs/preferences/cpufrequency/da.catkeys +++ /dev/null @@ -1,19 +0,0 @@ -1 danish x-vnd.Haiku-CPUFrequencyPref 3974849433 -CPU frequency status view CPU Frequency View CPU frekvens statusoversigt -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrekvens\n\tskrevet af Clemens Zeidler\n\tOphavsret 2009, Haiku, Inc.\n -Defaults Pref Window Standard indstillinger -Dynamic performance Status view Dynamisk ydeevne -Dynamic stepping CPU Frequency View Dynamisk acceleration -High performance Status view Høj ydeevne -Install replicant into Deskbar CPU Frequency View Indsæt replikant i Deskbar -Integration time [ms]: CPU Frequency View Integrationstid [ms]: -Launching the CPU frequency preflet failed.\n\nError: Status view Det var ikke muligt at starte CPU frekvens indstillings programmet.\n\nFejl: -Low energy Status view Lavt energiforbrug -Ok Status view OK -Open Speedstep preferences… Status view Åben Speedstep egenskaber… -Quit Status view Afslut -Revert Pref Window Tilbagestil -Set state Status view Sæt tilstand -Step up by CPU usage Color Step View Acceleration efter CPU belastning -Stepping policy CPU Frequency View Accelerationspolitik -Stepping policy: CPU Frequency View Accelerationspolitik: diff --git a/data/catalogs/preferences/cpufrequency/de.catkeys b/data/catalogs/preferences/cpufrequency/de.catkeys deleted file mode 100644 index 153a6e4ed5..0000000000 --- a/data/catalogs/preferences/cpufrequency/de.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 german x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view OK -CPU frequency status view CPU Frequency View CPU-Takt-Anzeige -OK Status view OK -Set state Status view Fester Takt -Defaults Pref Window Standardwerte -Install replicant into Deskbar CPU Frequency View Replikant in Deskbar installieren -Quit Status view Beenden -High performance Status view Hohe Leistung -Stepping policy CPU Frequency View Taktungsrichtlinie -Dynamic performance Status view Dynamische Leistung -Step up by CPU usage Color Step View Taktänderung ab CPU-Auslastung von: -Dynamic stepping CPU Frequency View Dynamische Taktung -Stepping policy: CPU Frequency View Taktungsrichtlinie: -Integration time [ms]: CPU Frequency View Messintervall [ms]: -Open Speedstep preferences… Status view Speedstep Einstellungen öffnen… -Revert Pref Window Anfangswerte -Low energy Status view Geringer Energieverbrauch -Launching the CPU frequency preflet failed.\n\nError: Status view Das CPU-Takt-Preflet konnte nicht gestartet werden.\n\nFehler: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPU-Takt\n\tGeschrieben von Clemens Zeidler.\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name CPU-Takt diff --git a/data/catalogs/preferences/cpufrequency/el.catkeys b/data/catalogs/preferences/cpufrequency/el.catkeys deleted file mode 100644 index 162b3e7df8..0000000000 --- a/data/catalogs/preferences/cpufrequency/el.catkeys +++ /dev/null @@ -1,20 +0,0 @@ -1 greek, modern (1453-) x-vnd.Haiku-CPUFrequencyPref 3079996868 -Ok Status view Εντάξει -CPU frequency status view CPU Frequency View Συχνότητα επεξεργαστή προβολή κατάστασης -Set state Status view Ορισμός κατάστασης -Defaults Pref Window Προεπιλογές -Install replicant into Deskbar CPU Frequency View Εγκατάσταση αντίγραφου στη Μπάρα εργασιών -Quit Status view Έξοδος -High performance Status view Υψηλή επίδοση -Stepping policy CPU Frequency View Πολιτική βηματισμού -Dynamic performance Status view Δυναμική επίδοση -Step up by CPU usage Color Step View Αύξηση με χρήση της CPU -Dynamic stepping CPU Frequency View Δυναμικός βηματισμός -Stepping policy: CPU Frequency View Πολιτική βηματισμού: -Integration time [ms]: CPU Frequency View Χρόνος ολοκλήρωσης [ms]: -Open Speedstep preferences… Status view Άνοιγμα προτιμήσεων Ταχύτητα βήματος -Revert Pref Window Επαναφορά -Low energy Status view Χαμηλή ενέργεια -Launching the CPU frequency preflet failed.\n\nError: Status view Απέτυχε η εκκίνηση του preflet Συχνότητα επεξεργαστή.\n\nΣφάλμα: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Συχνότητα Επεξεργαστή\n\tγράφτηκε από τον Clemens Zeidler\n\tΠνευματικά δικαιώματα 2009, Haiku, Inc.\n -CPUFrequency System name Συχνότητα της CPU diff --git a/data/catalogs/preferences/cpufrequency/eo.catkeys b/data/catalogs/preferences/cpufrequency/eo.catkeys deleted file mode 100644 index 0d1720550b..0000000000 --- a/data/catalogs/preferences/cpufrequency/eo.catkeys +++ /dev/null @@ -1,19 +0,0 @@ -1 esperanto x-vnd.Haiku-CPUFrequencyPref 3974849433 -CPU frequency status view CPU Frequency View Procesorofteca stato -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Procesorofteco\n\tskribita per Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -Defaults Pref Window Aprioraj -Dynamic performance Status view Dinamika rendimento -Dynamic stepping CPU Frequency View Dinamika akcelo -High performance Status view Plejbona rendimento -Install replicant into Deskbar CPU Frequency View Instali la kopianton en la Deskbar-on -Integration time [ms]: CPU Frequency View Integralada tempo [ms]: -Launching the CPU frequency preflet failed.\n\nError: Status view Malsukcesis lanĉi la procesorofteca prefileto.\n\nEraro: -Low energy Status view Malgranda energio -Ok Status view Bone -Open Speedstep preferences… Status view Malfermi la Speedstep-ajn agordojn… -Quit Status view Eliri -Revert Pref Window Malfari -Set state Status view Ŝanĝi staton -Step up by CPU usage Color Step View Akcelo per procesora uzo: -Stepping policy CPU Frequency View Akcela reĝimo -Stepping policy: CPU Frequency View Akcela reĝimo: diff --git a/data/catalogs/preferences/cpufrequency/fi.catkeys b/data/catalogs/preferences/cpufrequency/fi.catkeys deleted file mode 100644 index d75dd8d3b6..0000000000 --- a/data/catalogs/preferences/cpufrequency/fi.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 finnish x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Valmis -CPU frequency status view CPU Frequency View Suoritintaajuuden tilanäkymä -OK Status view Valmis -Set state Status view Aseta tila -Defaults Pref Window Oletukset -Install replicant into Deskbar CPU Frequency View Asenna kopio Työpöytäpalkkiin -Quit Status view Poistu -High performance Status view Korkea suorituskyky -Stepping policy CPU Frequency View Muutostapa -Dynamic performance Status view Dynaaminen suorituskyky -Step up by CPU usage Color Step View Muuta ylöspäin tämän suorittimen käyttömäärän kohdalla: -Dynamic stepping CPU Frequency View Dynaaminen muutos -Stepping policy: CPU Frequency View Muutostapa: -Integration time [ms]: CPU Frequency View Integraatioaika [ms]: -Open Speedstep preferences… Status view Avaa nopeusaskelvalinnat... -Revert Pref Window Palauta -Low energy Status view Matala energiankulutus -Launching the CPU frequency preflet failed.\n\nError: Status view Suoritintaajuuden asetusohjelman käynnistys epäonnistui.\n\nVirhe: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Suoritintaajuusasetukset\n\t tekijä: Clemens Zeidler\n\t Copyright 2009, Haiku, Inc.\n -CPUFrequency System name Suoritintaajuusasetukset diff --git a/data/catalogs/preferences/cpufrequency/fr.catkeys b/data/catalogs/preferences/cpufrequency/fr.catkeys deleted file mode 100644 index b0dbc2f29d..0000000000 --- a/data/catalogs/preferences/cpufrequency/fr.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 french x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Ok -CPU frequency status view CPU Frequency View Vue de l'état de la fréquence du CPU -OK Status view OK -Set state Status view Changer l'état -Defaults Pref Window Par défauts -Install replicant into Deskbar CPU Frequency View Installer le réplicant dans la Deskbar -Quit Status view Quitter -High performance Status view Haute performance -Stepping policy CPU Frequency View Stratégie d'accélération -Dynamic performance Status view Performance dynamique -Step up by CPU usage Color Step View Accélération selon l'activité CPU -Dynamic stepping CPU Frequency View Accélération dynamique -Stepping policy: CPU Frequency View Stratégie d'accélération : -Integration time [ms]: CPU Frequency View Temps d'intégration [ms] : -Open Speedstep preferences… Status view Ouvrir les préférences Speestep… -Revert Pref Window Rétablir -Low energy Status view Basse consommation -Launching the CPU frequency preflet failed.\n\nError: Status view Échec de lancement des préférences de fréquence du CPU.\n\nErreur : -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Fréquence CPU\n\tÉcrit par Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Fréquence CPU diff --git a/data/catalogs/preferences/cpufrequency/hi.catkeys b/data/catalogs/preferences/cpufrequency/hi.catkeys deleted file mode 100644 index 9911779ae4..0000000000 --- a/data/catalogs/preferences/cpufrequency/hi.catkeys +++ /dev/null @@ -1,20 +0,0 @@ -1 hindi x-vnd.Haiku-CPUFrequencyPref 3079996868 -Ok Status view ठीक है -CPU frequency status view CPU Frequency View सीपियु आवृत्ति स्थिति देखने के लिए -Set state Status view सेट स्टेट -Defaults Pref Window डिफ़ॉल्ट्स -Install replicant into Deskbar CPU Frequency View डेस्कबार में स्थापित रेप्लिका -Quit Status view छोड़ना -High performance Status view उच्च निष्पादन -Stepping policy CPU Frequency View कदम नीति -Dynamic performance Status view गतिशील प्रदर्शन -Step up by CPU usage Color Step View धीरे-धीरे सीपियु का उपयोग द्वारा चरण होना -Dynamic stepping CPU Frequency View गतिशील कदम -Stepping policy: CPU Frequency View कदम नीति: -Integration time [ms]: CPU Frequency View एकीकरण समय [एमएस]: -Open Speedstep preferences… Status view खुला स्पीद्स्तेप वरीयता ... -Revert Pref Window पलटें -Low energy Status view कम इनरजी -Launching the CPU frequency preflet failed.\n\nError: Status view लौन्चिंग सीपियु आवृत्ति शुभारंभ विफल.\n\nत्रुटि: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view सीपियु रेक़ुएन्क्य\n\t क्लेमेंस ज़िल्डर द्वारा त्व्रित्तें\n\t कोप्य्रिघ्त 2009, हाइकू, Inc.\n -CPUFrequency System name सीपियु की आवृत्ति diff --git a/data/catalogs/preferences/cpufrequency/hr.catkeys b/data/catalogs/preferences/cpufrequency/hr.catkeys deleted file mode 100644 index aa1f020eec..0000000000 --- a/data/catalogs/preferences/cpufrequency/hr.catkeys +++ /dev/null @@ -1,12 +0,0 @@ -1 croatian x-vnd.Haiku-CPUFrequencyPref 2355894194 -CPU frequency status view CPU Frequency View Pogled stanja frekvencije procesora -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Frekvencija procesora\n\tnapisao Clemens Zeidler\n\tAutoraska prava 2009, Haiku, Inc.\n -Defaults Pref Window Zadano -Dynamic performance Status view Dinamične performanse -High performance Status view Visoke performanse -Integration time [ms]: CPU Frequency View Vrijeme integracije [ms]: -Ok Status view U redu -Open Speedstep preferences… Status view Otvori Speedstep osobitosti... -Quit Status view Isključi -Revert Pref Window Preokreni -Set state Status view Postavi stanje diff --git a/data/catalogs/preferences/cpufrequency/hu.catkeys b/data/catalogs/preferences/cpufrequency/hu.catkeys deleted file mode 100644 index da469078c7..0000000000 --- a/data/catalogs/preferences/cpufrequency/hu.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 hungarian x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Rendben -CPU frequency status view CPU Frequency View Processzor-frekvencia állapota -OK Status view Rendben -Set state Status view Órajel beállítása -Defaults Pref Window Eredeti -Install replicant into Deskbar CPU Frequency View Replikáns telepítése az Asztalsávba -Quit Status view Kilépés -High performance Status view Nagy teljesítmény -Stepping policy CPU Frequency View Fokozatszabályozás -Dynamic performance Status view Dinamikus teljesítmény -Step up by CPU usage Color Step View Kiegyensúlyozott teljesítmény -Dynamic stepping CPU Frequency View Szabadon választható fokozatok -Stepping policy: CPU Frequency View Fokozatszabályozás: -Integration time [ms]: CPU Frequency View Alkalmazkodás ideje [ms]: -Open Speedstep preferences… Status view Fokozatszabályozási beállító megnyitása… -Revert Pref Window Visszaállít -Low energy Status view Alacsony energiafogyasztás -Launching the CPU frequency preflet failed.\n\nError: Status view Nem sikerült a Processzor-frekvencia elindítása.\n\nHiba: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Processzor-frekvencia\n\tÍrta: Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Processzor-frekvencia diff --git a/data/catalogs/preferences/cpufrequency/it.catkeys b/data/catalogs/preferences/cpufrequency/it.catkeys deleted file mode 100644 index dbbc4b643b..0000000000 --- a/data/catalogs/preferences/cpufrequency/it.catkeys +++ /dev/null @@ -1,19 +0,0 @@ -1 italian x-vnd.Haiku-CPUFrequencyPref 3974849433 -CPU frequency status view CPU Frequency View Stato frequenza della CPU -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Frequenza CPU\n\tscritto da Clemens Zeidler\n\tDiritti d'autore ©2009, Haiku, Inc.\n -Defaults Pref Window Predefiniti -Dynamic performance Status view Prestazioni dinamiche -Dynamic stepping CPU Frequency View Scalabilità dinamica -High performance Status view Prestazioni elevate -Install replicant into Deskbar CPU Frequency View Installa il replicante nella Deskbar -Integration time [ms]: CPU Frequency View Tempo di intervento [ms]: -Launching the CPU frequency preflet failed.\n\nError: Status view Impossibile far partire le preferenze frequenza CPU.\n\nErrore: -Low energy Status view Bassi consumi -Ok Status view Ok -Open Speedstep preferences… Status view Apri preferenze Speedstep… -Quit Status view Esci -Revert Pref Window Ripristina -Set state Status view Imposta stato -Step up by CPU usage Color Step View Scala in base all'utilizzo della CPU -Stepping policy CPU Frequency View Regole di scalabilità -Stepping policy: CPU Frequency View Regola scalabilità: diff --git a/data/catalogs/preferences/cpufrequency/ja.catkeys b/data/catalogs/preferences/cpufrequency/ja.catkeys deleted file mode 100644 index cd225eb8a4..0000000000 --- a/data/catalogs/preferences/cpufrequency/ja.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 japanese x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Ok -CPU frequency status view CPU Frequency View CPU 周波数ビュー -OK Status view OK -Set state Status view 設定 -Defaults Pref Window デフォルト -Install replicant into Deskbar CPU Frequency View レプリカントを Deskbar に表示 -Quit Status view 終了 -High performance Status view ハイパフォーマンス -Stepping policy CPU Frequency View ステッピングポリシー -Dynamic performance Status view ダイナミックパフォーマンス -Step up by CPU usage Color Step View CPU 負荷に応じてステップアップ -Dynamic stepping CPU Frequency View ダイナミックステッピング -Stepping policy: CPU Frequency View ステッピングポリシー -Integration time [ms]: CPU Frequency View 積分時間 [ms]: -Open Speedstep preferences… Status view Open Speedstep の設定… -Revert Pref Window 元に戻す -Low energy Status view 省電力モード -Launching the CPU frequency preflet failed.\n\nError: Status view CPU 周波数の起動に失敗しました。\n\nエラー: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPU 周波数\n\tClemens Zeidler 作\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name CPU 周波数 diff --git a/data/catalogs/preferences/cpufrequency/ko.catkeys b/data/catalogs/preferences/cpufrequency/ko.catkeys deleted file mode 100644 index da3fcad2cb..0000000000 --- a/data/catalogs/preferences/cpufrequency/ko.catkeys +++ /dev/null @@ -1,19 +0,0 @@ -1 korean x-vnd.Haiku-CPUFrequencyPref 3974849433 -CPU frequency status view CPU Frequency View CPU 주파수 상태 보기 -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPU주파수\n\tClemens Zeidler 작성\n\t저작권 2009, Haiku, Inc.\n -Defaults Pref Window 기본설정 -Dynamic performance Status view 동적 성능 -Dynamic stepping CPU Frequency View 동적 스테핑 -High performance Status view 고성능 -Install replicant into Deskbar CPU Frequency View 데스크바에 복제품 설치 -Integration time [ms]: CPU Frequency View 통합 시간 [ms]: -Launching the CPU frequency preflet failed.\n\nError: Status view CPU 주파수 제어판 불러오기 실패.\n\n에러: -Low energy Status view 저전력 -Ok Status view 확인 -Open Speedstep preferences… Status view 스피드스텝 설정 열기… -Quit Status view 나가기 -Revert Pref Window 되돌리기 -Set state Status view 상태 설정 -Step up by CPU usage Color Step View CPU 사용량에 따라 증가 -Stepping policy CPU Frequency View 스테핑 정책 -Stepping policy: CPU Frequency View 스테핑 정책: diff --git a/data/catalogs/preferences/cpufrequency/lt.catkeys b/data/catalogs/preferences/cpufrequency/lt.catkeys deleted file mode 100644 index 7e8f75462c..0000000000 --- a/data/catalogs/preferences/cpufrequency/lt.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 lithuanian x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Gerai -CPU frequency status view CPU Frequency View Centrinio procesoriaus dažnio indikatorius -OK Status view Gerai -Set state Status view Pasirinkta būsena -Defaults Pref Window Numatytai -Install replicant into Deskbar CPU Frequency View Patalpinti indikatorių pranešimų srityje -Quit Status view Baigti darbą -High performance Status view Didelis našumas -Stepping policy CPU Frequency View Centrinio procesoriaus veiksena -Dynamic performance Status view Prisitaikantis našumas -Step up by CPU usage Color Step View Našumo keitimo žingsnis -Dynamic stepping CPU Frequency View Prisitaikančio našumo derinimas -Stepping policy: CPU Frequency View Centrinio procesoriaus veiksena: -Integration time [ms]: CPU Frequency View Integravimosi laikas [ms]: -Open Speedstep preferences… Status view CP dažnio nuostatos… -Revert Pref Window Atšaukti pakeitimus -Low energy Status view Energijos taupymas -Launching the CPU frequency preflet failed.\n\nError: Status view Nepavyko atverti Centrinio procesoriaus dažnio nuostatų lango.\n\nKlaida: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Centrinio procesoriaus dažnis\n\tsukurtas Clemens'o Zeidler'io\n\t© 2009, Haiku, Inc.\n -CPUFrequency System name Centrinio procesoriaus dažnis diff --git a/data/catalogs/preferences/cpufrequency/nb.catkeys b/data/catalogs/preferences/cpufrequency/nb.catkeys deleted file mode 100644 index 3a594c7606..0000000000 --- a/data/catalogs/preferences/cpufrequency/nb.catkeys +++ /dev/null @@ -1,19 +0,0 @@ -1 bokmål, norwegian; norwegian bokmål x-vnd.Haiku-CPUFrequencyPref 1579805342 -Ok Status view OK -CPU frequency status view CPU Frequency View Visning av CPU-status -Set state Status view Sett tilstand -Defaults Pref Window Standardinnstillinger -Install replicant into Deskbar CPU Frequency View Installer replikanten i Deskbar -Quit Status view Avslutt -High performance Status view Høy ytelse -Stepping policy CPU Frequency View Stepping-policy -Dynamic performance Status view Dynamisk ytelse -Dynamic stepping CPU Frequency View Dynamisk stepping -Stepping policy: CPU Frequency View Stepping-policy -Integration time [ms]: CPU Frequency View Integrasjonstid [ms]: -Open Speedstep preferences… Status view Åpne Speedstep-innstillinger -Revert Pref Window Tilbakestill -Low energy Status view Lavenergi -Launching the CPU frequency preflet failed.\n\nError: Status view Feil ved lasting av Prosessorhastighet. \n\nFeil: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tskrevet av Clemens Zeidler\n\tOpphavsrett 2009, Haiku, Inc.\n -CPUFrequency System name Prosessorhastighet diff --git a/data/catalogs/preferences/cpufrequency/nl.catkeys b/data/catalogs/preferences/cpufrequency/nl.catkeys deleted file mode 100644 index 4367c89acb..0000000000 --- a/data/catalogs/preferences/cpufrequency/nl.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 dutch; flemish x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Oké -CPU frequency status view CPU Frequency View Weergave CPU-frequentiestatus -OK Status view Ok -Set state Status view Toestand instellen -Defaults Pref Window Standaardwaarden -Install replicant into Deskbar CPU Frequency View Replicant in Deskbar installeren -Quit Status view Afsluiten -High performance Status view Hoge prestaties -Stepping policy CPU Frequency View Stapbeleid -Dynamic performance Status view Dynamische prestaties -Step up by CPU usage Color Step View Opvoeren door CPU-gebruik -Dynamic stepping CPU Frequency View Dynamisch opvoeren -Stepping policy: CPU Frequency View Opvoerbeleid: -Integration time [ms]: CPU Frequency View Integratietijd [ms]: -Open Speedstep preferences… Status view Speedstep-voorkeuren openen… -Revert Pref Window Herstellen -Low energy Status view Lage energie -Launching the CPU frequency preflet failed.\n\nError: Status view Starten van de CPU-frequentiepreflet is mislukt.\n\nFout: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tdoor Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name CPUFrequency diff --git a/data/catalogs/preferences/cpufrequency/pl.catkeys b/data/catalogs/preferences/cpufrequency/pl.catkeys deleted file mode 100644 index 1cba58892d..0000000000 --- a/data/catalogs/preferences/cpufrequency/pl.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 polish x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Ok -CPU frequency status view CPU Frequency View Widok statusu częstotliwości taktowania procesora -OK Status view OK -Set state Status view Ustaw stan -Defaults Pref Window Przywróć ustawienia domyślne -Install replicant into Deskbar CPU Frequency View Zainstaluj replikant na Panelu Pulpitu -Quit Status view Zamknij -High performance Status view Wysoka wydajność -Stepping policy CPU Frequency View Polityka skalowania procesora -Dynamic performance Status view Dynamicznie dopasowana wydajność -Step up by CPU usage Color Step View Ustawiaj taktowanie w zależności od obciążenia procesora -Dynamic stepping CPU Frequency View Dynamiczne skalowanie -Stepping policy: CPU Frequency View Polityka skalowania: -Integration time [ms]: CPU Frequency View Czas integracji (w milisekundach): -Open Speedstep preferences… Status view Otwórz ustawienia Speedstep… -Revert Pref Window Przywróć poprzednie ustawienia -Low energy Status view Oszczędna -Launching the CPU frequency preflet failed.\n\nError: Status view Uruchomienie prefletu częstotliwości procesora zakończyło się niepowodzeniem.\n\nBłąd: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tnapisane przez Clemensa Zeidlera\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Częstotliwość procesora diff --git a/data/catalogs/preferences/cpufrequency/pt.catkeys b/data/catalogs/preferences/cpufrequency/pt.catkeys deleted file mode 100644 index ae2db82ce7..0000000000 --- a/data/catalogs/preferences/cpufrequency/pt.catkeys +++ /dev/null @@ -1,12 +0,0 @@ -1 portuguese x-vnd.Haiku-CPUFrequencyPref 2133997991 -Defaults Pref Window Predefenições -Dynamic stepping CPU Frequency View Aceleração Dinâmica -Install replicant into Deskbar CPU Frequency View Instalar o Replicant na Deskbar -Integration time [ms]: CPU Frequency View Tempo de integração [ms]: -Ok Status view OK -Quit Status view Sair -Revert Pref Window Reverter -Set state Status view Defenir estado -Step up by CPU usage Color Step View Variação de velocidade com base no uso do CPU -Stepping policy CPU Frequency View Política de Aceleração -Stepping policy: CPU Frequency View Política de Aceleração: diff --git a/data/catalogs/preferences/cpufrequency/pt_BR.catkeys b/data/catalogs/preferences/cpufrequency/pt_BR.catkeys deleted file mode 100644 index 6744324be8..0000000000 --- a/data/catalogs/preferences/cpufrequency/pt_BR.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 portuguese (brazil) x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view Ok -CPU frequency status view CPU Frequency View Status da frequência do CPU -OK Status view OK -Set state Status view Definir estado -Defaults Pref Window Padrões -Install replicant into Deskbar CPU Frequency View Instalar o replicante na Mesa de Trabalho -Quit Status view Sair -High performance Status view Alto desempenho -Stepping policy CPU Frequency View Política de stepping -Dynamic performance Status view Desempenho dinâmico -Step up by CPU usage Color Step View Aumentar desempenho por uso de CPU -Dynamic stepping CPU Frequency View Stepping dinâmico -Stepping policy: CPU Frequency View Política de stepping: -Integration time [ms]: CPU Frequency View Tempo de integração [ms]: -Open Speedstep preferences… Status view Preferências de Open Speedstep… -Revert Pref Window Reverter -Low energy Status view Baixa energia -Launching the CPU frequency preflet failed.\n\nError: Status view Não foi possível abrir a Frequência de CPU.\n\nErro: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view A Frequência do CPU\n\tfoi escrito por Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Frequência do CPU diff --git a/data/catalogs/preferences/cpufrequency/pt_br.catkeys b/data/catalogs/preferences/cpufrequency/pt_br.catkeys deleted file mode 100644 index e8d674debe..0000000000 --- a/data/catalogs/preferences/cpufrequency/pt_br.catkeys +++ /dev/null @@ -1,8 +0,0 @@ -1 brazilian_portuguese x-vnd.Haiku-CPUFrequencyPref 1149286520 -CPU frequency status view CPU Frequency View Status da frequência do CPU -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view A Frequência do CPU\n\tfoi escrito por Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -Defaults Pref Window Padrões -Install replicant into Deskbar CPU Frequency View Instalar o replicante na Mesa de Trabalho -Launching the CPU frequency preflet failed.\n\nError: Status view Não foi possível abrir a Frequência de CPU.\n\nErro: -Low energy Status view Baixa energia -Set state Status view Definir estado diff --git a/data/catalogs/preferences/cpufrequency/ro.catkeys b/data/catalogs/preferences/cpufrequency/ro.catkeys deleted file mode 100644 index 38acfb5c61..0000000000 --- a/data/catalogs/preferences/cpufrequency/ro.catkeys +++ /dev/null @@ -1,20 +0,0 @@ -1 romanian x-vnd.Haiku-CPUFrequencyPref 3079996868 -Ok Status view Ok -CPU frequency status view CPU Frequency View Vizualizare a stării frecvenței CPU -Set state Status view Stabilește stare -Defaults Pref Window Implicite -Install replicant into Deskbar CPU Frequency View Instalează copie pe Deskbar -Quit Status view Părăsește -High performance Status view Performanță înaltă -Stepping policy CPU Frequency View Regulă de accelerare -Dynamic performance Status view Performanță dinamică -Step up by CPU usage Color Step View Accelerare după utilizarea CPU -Dynamic stepping CPU Frequency View Accelerare dinamică -Stepping policy: CPU Frequency View Regulă de accelerare: -Integration time [ms]: CPU Frequency View Timp de integrare [ms]: -Open Speedstep preferences… Status view Deschide preferințe viteză… -Revert Pref Window Revenire -Low energy Status view Energie scăzută -Launching the CPU frequency preflet failed.\n\nError: Status view Pornirea prefletului frecvență CPU a eșuat.\n\nEroare: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view FrecvențăCPU\n\tscris de Clemens Zeidler\n\tDrept de autor 2009, Haiku, Inc.\n -CPUFrequency System name FrecvențăCPU diff --git a/data/catalogs/preferences/cpufrequency/ru.catkeys b/data/catalogs/preferences/cpufrequency/ru.catkeys deleted file mode 100644 index 850cc69d7d..0000000000 --- a/data/catalogs/preferences/cpufrequency/ru.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 russian x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view ОК -CPU frequency status view CPU Frequency View Просмотр состояния частоты ЦП -OK Status view OK -Set state Status view Выбрать состояние -Defaults Pref Window По умолчанию -Install replicant into Deskbar CPU Frequency View Установить репликант в Deskbar -Quit Status view Выход -High performance Status view Высокая производительность -Stepping policy CPU Frequency View Настройка шага изменения частоты ЦП -Dynamic performance Status view Динамическая производительность -Step up by CPU usage Color Step View Шаг вверх по производительности ЦП: -Dynamic stepping CPU Frequency View Динамическое изменение -Stepping policy: CPU Frequency View Настройка шага изменения частоты ЦП: -Integration time [ms]: CPU Frequency View Время интеграции [мс]: -Open Speedstep preferences… Status view Настроить частоту процессора… -Revert Pref Window Вернуть -Low energy Status view Экономия энергии -Launching the CPU frequency preflet failed.\n\nError: Status view Не удалось запустить настройки частоты ЦП.\n\nОшибка: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tразработал Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Частота процессора diff --git a/data/catalogs/preferences/cpufrequency/sk.catkeys b/data/catalogs/preferences/cpufrequency/sk.catkeys deleted file mode 100644 index 87f5b3572d..0000000000 --- a/data/catalogs/preferences/cpufrequency/sk.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 slovak x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view OK -CPU frequency status view CPU Frequency View Zobrazenie stavy frekvencie CPU -OK Status view OK -Set state Status view Nastaviť stav -Defaults Pref Window Predvoľby -Install replicant into Deskbar CPU Frequency View Nainštalovať replikant do Panelu -Quit Status view Ukončiť -High performance Status view Vysoký výkon -Stepping policy CPU Frequency View Politika krokovania -Dynamic performance Status view Dynamický výkon -Step up by CPU usage Color Step View Krokovať podľa využitia CPU -Dynamic stepping CPU Frequency View Dynamické krokovanie -Stepping policy: CPU Frequency View Politika krokovania: -Integration time [ms]: CPU Frequency View Čas integrácie [ms]: -Open Speedstep preferences… Status view Otvoriť nastavenia Speedstep… -Revert Pref Window Vrátiť -Low energy Status view Nízky odber -Launching the CPU frequency preflet failed.\n\nError: Status view Spustenie konfiguračného apletu Frekvencia CPU zlyhalo.\n\nChyba: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPUFrequency\n\tnapísal Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Frekvencia CPU diff --git a/data/catalogs/preferences/cpufrequency/sv.catkeys b/data/catalogs/preferences/cpufrequency/sv.catkeys deleted file mode 100644 index 8762227b89..0000000000 --- a/data/catalogs/preferences/cpufrequency/sv.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 swedish x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view OK -CPU frequency status view CPU Frequency View Statusvy för ProcessorFrekvens -OK Status view OK -Set state Status view Ändra läge -Defaults Pref Window Förval -Install replicant into Deskbar CPU Frequency View Installera replikant i Deskbar -Quit Status view Avsluta -High performance Status view Hög prestanda -Stepping policy CPU Frequency View Accelerationspolicy -Dynamic performance Status view Dynamisk prestanda -Step up by CPU usage Color Step View Öka vid CPU-belastning -Dynamic stepping CPU Frequency View Dynamisk acceleration -Stepping policy: CPU Frequency View Accelerationspolicy: -Integration time [ms]: CPU Frequency View Integreringstid -Open Speedstep preferences… Status view Öppna inställningar för ProcessorFrekvens… -Revert Pref Window Återgå -Low energy Status view Lågenergiläge -Launching the CPU frequency preflet failed.\n\nError: Status view Misslyckades med att starta inställningsprogrammet ProcessorFrekvens.\n\nFel: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view ProcessorFrekvens\n\tskriven av Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name ProcessorFrekvens diff --git a/data/catalogs/preferences/cpufrequency/uk.catkeys b/data/catalogs/preferences/cpufrequency/uk.catkeys deleted file mode 100644 index 58dd878c6b..0000000000 --- a/data/catalogs/preferences/cpufrequency/uk.catkeys +++ /dev/null @@ -1,20 +0,0 @@ -1 ukrainian x-vnd.Haiku-CPUFrequencyPref 3079996868 -Ok Status view Гаразд -CPU frequency status view CPU Frequency View Статистика частоти ЦП -Set state Status view Встановити стан -Defaults Pref Window По замовчуванню -Install replicant into Deskbar CPU Frequency View Встановити реплікант в Deskbar -Quit Status view Вийти -High performance Status view Висока продуктивність -Stepping policy CPU Frequency View Настройка кроку зміни частоти ЦП -Dynamic performance Status view Динамічна продуктивність -Step up by CPU usage Color Step View Крок вгору по продуктивності ЦП: -Dynamic stepping CPU Frequency View Динамічна зміна -Stepping policy: CPU Frequency View Настройка кроку зміни частоти ЦП: -Integration time [ms]: CPU Frequency View Час інтеграції [мс]: -Open Speedstep preferences… Status view Відкриття покрокової настройки -Revert Pref Window Повернути -Low energy Status view Мале споживання -Launching the CPU frequency preflet failed.\n\nError: Status view Програму Частота процесора не вдалося запустити.\n\nПомилка: -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view Частота процесора\n\Автор Клеменс Зейдлер\n\tCopyright 2009, Haiku, Inc.\n -CPUFrequency System name Частота процесора diff --git a/data/catalogs/preferences/cpufrequency/zh-Hans.catkeys b/data/catalogs/preferences/cpufrequency/zh-Hans.catkeys deleted file mode 100644 index 215219bf72..0000000000 --- a/data/catalogs/preferences/cpufrequency/zh-Hans.catkeys +++ /dev/null @@ -1,20 +0,0 @@ -1 english x-vnd.Haiku-CPUFrequencyPref 3079996868 -Ok Status view 确定 -CPU frequency status view CPU Frequency View CPU 状态视图 -Set state Status view 状态设置 -Defaults Pref Window 默认 -Install replicant into Deskbar CPU Frequency View 添加 CPU 监视器到桌面栏 -Quit Status view 退出 -High performance Status view 高性能 -Stepping policy CPU Frequency View 步进策略 -Dynamic performance Status view 动态性能 -Step up by CPU usage Color Step View CPU 步进设置 -Dynamic stepping CPU Frequency View 动态步进 -Stepping policy: CPU Frequency View 步进策略： -Integration time [ms]: CPU Frequency View 采样时间 [ms]： -Open Speedstep preferences… Status view 打开省电设置… -Revert Pref Window 取消 -Low energy Status view 低能耗 -Launching the CPU frequency preflet failed.\n\nError: Status view 启动 CPU 状态通知失败。\n\n错误信息： -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPU 监视器\n\t编写者：Clemens Zeidler\n\tCopyright 2009，Haiku .Inc。\n -CPUFrequency System name CPU 监视器 diff --git a/data/catalogs/preferences/cpufrequency/zh_Hans.catkeys b/data/catalogs/preferences/cpufrequency/zh_Hans.catkeys deleted file mode 100644 index c93c63c420..0000000000 --- a/data/catalogs/preferences/cpufrequency/zh_Hans.catkeys +++ /dev/null @@ -1,21 +0,0 @@ -1 english x-vnd.Haiku-CPUFrequencyPref 1007318924 -Ok Status view 确定 -CPU frequency status view CPU Frequency View CPU 状态视图 -OK Status view 确定 -Set state Status view 状态设置 -Defaults Pref Window 默认 -Install replicant into Deskbar CPU Frequency View 添加 CPU 监视器到桌面栏 -Quit Status view 退出 -High performance Status view 高性能 -Stepping policy CPU Frequency View 步进策略 -Dynamic performance Status view 动态性能 -Step up by CPU usage Color Step View CPU 步进设置 -Dynamic stepping CPU Frequency View 动态步进 -Stepping policy: CPU Frequency View 步进策略： -Integration time [ms]: CPU Frequency View 采样时间 [ms]： -Open Speedstep preferences… Status view 打开省电设置… -Revert Pref Window 取消 -Low energy Status view 低能耗 -Launching the CPU frequency preflet failed.\n\nError: Status view 启动 CPU 状态通知失败。\n\n错误信息： -CPUFrequency\n\twritten by Clemens Zeidler\n\tCopyright 2009, Haiku, Inc.\n Status view CPU 监视器\n\t编写者：Clemens Zeidler\n\tCopyright 2009，Haiku .Inc。\n -CPUFrequency System name CPU 监视器 diff --git a/src/preferences/Jamfile b/src/preferences/Jamfile index 99c59fb3c8..05a3858233 100644 --- a/src/preferences/Jamfile +++ b/src/preferences/Jamfile @@ -3,7 +3,6 @@ SubDir HAIKU_TOP src preferences ; SubInclude HAIKU_TOP src preferences appearance ; SubInclude HAIKU_TOP src preferences backgrounds ; SubInclude HAIKU_TOP src preferences bluetooth ; -SubInclude HAIKU_TOP src preferences cpufrequency ; SubInclude HAIKU_TOP src preferences datatranslations ; SubInclude HAIKU_TOP src preferences deskbar ; #SubInclude HAIKU_TOP src preferences dun ; diff --git a/src/preferences/cpufrequency/CPUFrequencyView.cpp b/src/preferences/cpufrequency/CPUFrequencyView.cpp deleted file mode 100644 index b0e1adf6c2..0000000000 --- a/src/preferences/cpufrequency/CPUFrequencyView.cpp +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright 2009-2010, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - - -#include "CPUFrequencyView.h" - -#include "StatusView.h" - -#include -#include -#include -#include -#include -#include -#include - - -#undef B_TRANSLATION_CONTEXT -#define B_TRANSLATION_CONTEXT "CPU Frequency View" - - -const char* kCPUFreqPreferencesFile = "CPUFrequency"; -const char* kPrefSignature = "application/x-vnd.Haiku-CPUFrequencyPref"; -const char* kPreferencesFileName = "CPUFrequency"; - -const uint32 kInstallIntoDeskbar = '&iid'; -const uint32 kIntegrationTimeChanged = '&itc'; - -const bigtime_t kMilliSecond = 1000; - - -CPUFrequencyView::CPUFrequencyView(BRect frame, - PreferencesStorage* storage) - : - BView(frame, "CPUFrequencyView", B_FOLLOW_NONE, B_WILL_DRAW), - fStorage(storage) -{ - BGroupLayout* mainLayout = new BGroupLayout(B_VERTICAL); - SetLayout(mainLayout); - mainLayout->SetSpacing(10); - SetViewColor(ui_color(B_PANEL_BACKGROUND_COLOR)); - - // stepping policy - BRect rect = Bounds(); - rect.InsetBy(5, 5); - BBox *policyBox = new BBox(rect, "policyBox"); - policyBox->SetLabel(B_TRANSLATE("Stepping policy")); - BGroupLayout* policyLayout = new BGroupLayout(B_VERTICAL); - policyLayout->SetInsets(10, policyBox->TopBorderOffset() * 2 + 10, 10, 10); - policyLayout->SetSpacing(10); - policyBox->SetLayout(policyLayout); - mainLayout->AddView(policyBox); - - fPolicyMenu = new BMenu(B_TRANSLATE("Stepping policy: ")); - BMenuField *menuField = new BMenuField("", fPolicyMenu); - policyLayout->AddView(menuField); - - // dynamic stepping - BBox *dynamicBox = new BBox(rect, "dynamicBox"); - dynamicBox->SetLabel(B_TRANSLATE("Dynamic stepping")); - BGroupLayout* dynamicLayout = new BGroupLayout(B_VERTICAL); - dynamicLayout->SetInsets(B_USE_DEFAULT_SPACING, - dynamicBox->TopBorderOffset() * 2 + 10, - B_USE_DEFAULT_SPACING, B_USE_DEFAULT_SPACING); - dynamicLayout->SetSpacing(B_USE_DEFAULT_SPACING); - dynamicBox->SetLayout(dynamicLayout); - mainLayout->AddView(dynamicBox); - - fColorStepView = new ColorStepView(frame); - fColorStepView->SetFrequencys(fDriverInterface.GetCpuFrequencyStates()); - - fIntegrationTime = new BTextControl(BRect(0, 0, Bounds().Width(), 10), - "intergal", B_TRANSLATE("Integration time [ms]: "), "", - new BMessage(kIntegrationTimeChanged)); - fIntegrationTime->SetDivider(Bounds().Width() * 3 / 4); - fIntegrationTime->SetAlignment(B_ALIGN_RIGHT, B_ALIGN_LEFT); - - dynamicLayout->AddView(fColorStepView); - dynamicLayout->AddView(fIntegrationTime); - - // status view - BBox *statusBox = new BBox(rect, "statusBox"); - statusBox->SetLabel(B_TRANSLATE("CPU frequency status view")); - BGroupLayout* statusLayout = new BGroupLayout(B_HORIZONTAL); - statusLayout->SetInsets(10, statusBox->TopBorderOffset() * 2 + 10, 10, 10); - statusLayout->SetSpacing(10); - statusBox->SetLayout(statusLayout); - mainLayout->AddView(statusBox); - - fStatusView = new StatusView(BRect(0, 0, 5, 5), false, fStorage); - fStatusView->ShowPopUpMenu(false); - - fInstallButton = new BButton("installButton", - B_TRANSLATE("Install replicant into Deskbar"), - new BMessage(kInstallIntoDeskbar)); - - statusLayout->AddView(fStatusView); - statusLayout->AddItem(BSpaceLayoutItem::CreateGlue()); - statusLayout->AddView(fInstallButton); -} - - -void -CPUFrequencyView::MessageReceived(BMessage* message) -{ - freq_preferences* pref = fStorage->GetPreferences(); - bool configChanged = false; - - switch (message->what) { - case kUpdatedPreferences: - fStatusView->MessageReceived(message); - if (pref->mode == DYNAMIC) { - fColorStepView->SetEnabled(true); - fIntegrationTime->SetEnabled(true); - } - else { - fColorStepView->SetEnabled(false); - fIntegrationTime->SetEnabled(false); - } - configChanged = true; - break; - - case kSteppingChanged: - // from ColorStepView - pref->stepping_threshold = fColorStepView->GetSliderPosition(); - fStorage->SavePreferences(); - configChanged = true; - break; - - case kIntegrationTimeChanged: - _ReadIntegrationTime(); - fStorage->SavePreferences(); - configChanged = true; - break; - - case kInstallIntoDeskbar: - _InstallReplicantInDeskbar(); - break; - - case kRevertMsg: - case kDefaultMsg: - fStatusView->UpdateCPUFreqState(); - _UpdateViews(); - break; - default: - BView::MessageReceived(message); - } - - if (configChanged) - Window()->PostMessage(kConfigChangedMsg); -} - - -void -CPUFrequencyView::AttachedToWindow() -{ - fFrequencyMenu = new FrequencyMenu(fPolicyMenu, this, fStorage, - &fDriverInterface); - AddFilter(fFrequencyMenu); - - fColorStepView->SetTarget(this); - fIntegrationTime->SetTarget(this); - fInstallButton->SetTarget(this); - - _UpdateViews(); -} - - -void -CPUFrequencyView::DetachedFromWindow() -{ - // emty menu for the case the view is attached again - while (true) { - BMenuItem* item = fPolicyMenu->RemoveItem(int32(0)); - if (!item) - break; - delete item; - } - if (RemoveFilter(fFrequencyMenu)) - delete fFrequencyMenu; - - _ReadIntegrationTime(); -} - - -status_t -our_image(image_info& image) -{ - int32 cookie = 0; - while (get_next_image_info(B_CURRENT_TEAM, &cookie, &image) == B_OK) { - if ((char *)our_image >= (char *)image.text - && (char *)our_image <= (char *)image.text + image.text_size) - return B_OK; - } - - return B_ERROR; -}; - - -void -CPUFrequencyView::_InstallReplicantInDeskbar() -{ - image_info info; - entry_ref ref; - if (our_image(info) == B_OK - && get_ref_for_path(info.name, &ref) == B_OK) { - BDeskbar deskbar; - deskbar.AddItem(&ref); - } -} - - -void -CPUFrequencyView::_UpdateViews() -{ - fFrequencyMenu->UpdateMenu(); - freq_preferences* pref = fStorage->GetPreferences(); - fColorStepView->SetSliderPosition(pref->stepping_threshold); - if (pref->mode == DYNAMIC) { - fColorStepView->SetEnabled(true); - fIntegrationTime->SetEnabled(true); - } - else { - fColorStepView->SetEnabled(false); - fIntegrationTime->SetEnabled(false); - } - - BString out; - out << pref->integration_time / kMilliSecond; - fIntegrationTime->SetText(out.String()); -} - - -void -CPUFrequencyView::_ReadIntegrationTime() -{ - freq_preferences* pref = fStorage->GetPreferences(); - bigtime_t integration_time = atoi(fIntegrationTime->Text()) * kMilliSecond; - if (integration_time == 0) { - BString out; - out << pref->integration_time / kMilliSecond; - fIntegrationTime->SetText(out.String()); - } - else { - pref->integration_time = integration_time; - } -} diff --git a/src/preferences/cpufrequency/CPUFrequencyView.h b/src/preferences/cpufrequency/CPUFrequencyView.h deleted file mode 100644 index 1d9a164ab9..0000000000 --- a/src/preferences/cpufrequency/CPUFrequencyView.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - -#ifndef CPUFREQUENCYVIEW_h -#define CPUFREQUENCYVIEW_h - -#include "DriverInterface.h" -#include "ColorStepView.h" -#include "PreferencesWindow.h" - -#include -#include -#include -#include - - -extern const char* kPrefSignature; -extern const char* kPreferencesFileName; - -enum stepping_mode { - DYNAMIC, - PERFORMANCE, - LOW_ENERGIE, - CUSTOM -}; - - -struct freq_preferences -{ - bool IsEqual(const freq_preferences& prefs) { - if (mode == prefs.mode && custom_stepping == prefs.custom_stepping - && stepping_threshold == prefs.stepping_threshold - && integration_time == prefs.integration_time) - return true; - return false; - } - - // stepping mode - stepping_mode mode; - int16 custom_stepping; - - // dynamic stepping - float stepping_threshold; - bigtime_t integration_time; -}; - - -const freq_preferences kDefaultPreferences = -{ - DYNAMIC, - -1, - 0.25, - 500000 //half second -}; - - -class FrequencyMenu; -class StatusView; - -class CPUFrequencyView : public BView -{ - public: - CPUFrequencyView(BRect frame, - PreferencesStorage* storage); - virtual void MessageReceived(BMessage* message); - - virtual void AttachedToWindow(); - virtual void DetachedFromWindow(); - - private: - void _InstallReplicantInDeskbar(); - void _UpdateViews(); - void _ReadIntegrationTime(); - - BMenu* fPolicyMenu; - FrequencyMenu* fFrequencyMenu; - ColorStepView* fColorStepView; - BTextControl* fIntegrationTime; - StatusView* fStatusView; - BButton* fInstallButton; - - PreferencesStorage* fStorage; - CPUFreqDriverInterface fDriverInterface; -}; - -#endif diff --git a/src/preferences/cpufrequency/ColorStepView.cpp b/src/preferences/cpufrequency/ColorStepView.cpp deleted file mode 100644 index 7b5a9caefd..0000000000 --- a/src/preferences/cpufrequency/ColorStepView.cpp +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - -#include "ColorStepView.h" - -#include - -#include -#include -#include - - -#undef B_TRANSLATION_CONTEXT -#define B_TRANSLATION_CONTEXT "Color Step View" - - -const int32 kColorBarHeight = 15; - -const uint32 kMSGSliderChanged = '&slc'; - - -ColorStepView::ColorStepView(BRect frame) - : BControl(frame, "ColorStepView", "", new BMessage(kSteppingChanged), - B_FOLLOW_ALL, B_WILL_DRAW), - fOffScreenView(NULL), - fOffScreenBitmap(NULL) -{ - fPerformanceList = new PerformanceList(20, true); - - fSliderPosition = 0; - fNSteps = 0; - fLowFreqColor.red = 0; - fLowFreqColor.blue = 255; - fLowFreqColor.green = 0; - - fHighFreqColor.red = 255; - fHighFreqColor.blue = 0; - fHighFreqColor.green = 0; - - fMinFrequencyLabel = "? MHz"; - fMaxFrequencyLabel = "? MHz"; - _InitView(); -} - - -ColorStepView::~ColorStepView() -{ - delete fPerformanceList; -} - - -void -ColorStepView::AttachedToWindow() -{ - fSlider->SetTarget(this); - - if (!fOffScreenView) { - fOffScreenView = new BView(Bounds(), "", B_FOLLOW_ALL, B_WILL_DRAW); - } - if (!fOffScreenBitmap) { - fOffScreenBitmap = new BBitmap(Bounds(), B_CMAP8, true, false); - Window()->Lock(); - if (fOffScreenBitmap && fOffScreenView) - fOffScreenBitmap->AddChild(fOffScreenView); - Window()->Unlock(); - } -} - - -void -ColorStepView::DetachedFromWindow() -{ - BView::DetachedFromWindow(); - - if (fOffScreenBitmap) { - delete fOffScreenBitmap; - fOffScreenBitmap = NULL; - fOffScreenView = NULL; - } -} - - -void -ColorStepView::FrameResized(float w,float h) -{ - BView::FrameResized(w, h); - - BRect bounds(Bounds()); - - if (bounds.right <= 0.0f || bounds.bottom <= 0.0f) - return; - - if (fOffScreenBitmap) { - fOffScreenBitmap->RemoveChild(fOffScreenView); - delete fOffScreenBitmap; - - fOffScreenView->ResizeTo(bounds.Width(), bounds.Height()); - - fOffScreenBitmap = new BBitmap(Bounds(), B_RGBA32, true, false); - fOffScreenBitmap->AddChild(fOffScreenView); - } - - Invalidate(); -} - - -void -ColorStepView::GetPreferredSize(float *width, float *height) -{ - *width = Frame().Width(); - font_height fontHeight; - GetFontHeight(&fontHeight); - - *height = fSlider->Frame().Height(); - *height += kColorBarHeight; - *height += fontHeight.descent + fontHeight.ascent + 5; -} - - -void -ColorStepView::Draw(BRect updateRect) -{ - BView *view = NULL; - if(fOffScreenView){ - view = fOffScreenView; - } - else{ - view = this; - } - - if (!fOffScreenBitmap || !fOffScreenBitmap->Lock()) - return; - view->SetLowColor(ui_color(B_PANEL_BACKGROUND_COLOR)); - - view->SetHighColor(ui_color(B_PANEL_BACKGROUND_COLOR)); - view->FillRect(updateRect); - - BRect colorBarRect = fSlider->BarFrame(); - colorBarRect.top = 0; - colorBarRect.bottom = kColorBarHeight; - colorBarRect.OffsetTo(colorBarRect.left, fSlider->Frame().bottom); - - float pos = 0.0; - for (int i = fPerformanceList->CountItems() - 1; i >= 0 ; i--) { - performance_step* perfState = fPerformanceList->ItemAt(i); - - float nextPos = perfState->cpu_usage; - float width = colorBarRect.Width(); - - BRect subRect(colorBarRect); - subRect.left += pos * width; - subRect.right = colorBarRect.left + nextPos * width; - - view->SetHighColor(perfState->color); - view->FillRect(subRect); - - pos = nextPos; - } - // draw label - if (IsEnabled()) - view->SetHighColor(0, 0, 0); - else - view->SetHighColor(tint_color(LowColor(), B_DISABLED_LABEL_TINT)); - - font_height fontHeight; - GetFontHeight(&fontHeight); - float totalFontHeight = fontHeight.descent + fontHeight.ascent; - - view->DrawString(fMinFrequencyLabel.String(), - BPoint(0.0, - colorBarRect.bottom + totalFontHeight + 5)); - - view->DrawString(fMaxFrequencyLabel.String(), - BPoint(Bounds().right - - StringWidth(fMaxFrequencyLabel.String()), - colorBarRect.bottom + totalFontHeight + 5)); - - // blit bitmap - view->Sync(); - fOffScreenBitmap->Unlock(); - DrawBitmap(fOffScreenBitmap, B_ORIGIN); - - BView::Draw(updateRect); -} - - -void -ColorStepView::MessageReceived(BMessage *message) -{ - switch (message->what) { - case kMSGSliderChanged: - fSliderPosition = fSlider->Position(); - _CalculatePerformanceSteps(); - Invalidate(); - break; - case kSteppingChanged: - Invoke(); - break; - default: - BView::MessageReceived(message); - break; - } -} - - -void -ColorStepView::SetEnabled(bool enabled) -{ - fSlider->SetEnabled(enabled); - BControl::SetEnabled(enabled); -} - - -void -ColorStepView::SetFrequencys(StateList *list) -{ - fStateList = list; - fNSteps = fStateList->CountItems(); - if (fNSteps >= 2) { - float minFreq = fStateList->ItemAt(fNSteps - 1)->frequency; - float maxFreq = fStateList->ItemAt(0)->frequency; - fMinFrequencyLabel = CreateFrequencyString(minFreq); - fMaxFrequencyLabel = CreateFrequencyString(maxFreq); - } - - // fit size of fPerformanceList - int32 perfNumber = fPerformanceList->CountItems(); - if (perfNumber < fNSteps) { - for (int i = 0; i < fNSteps - perfNumber; i++) - fPerformanceList->AddItem(new performance_step); - } else { - for (int i = 0; i < perfNumber - fNSteps; i++) - fPerformanceList->RemoveItemAt(0); - } - // and fill the list - _CalculatePerformanceSteps(); - -} - - -PerformanceList* -ColorStepView::GetPerformanceSteps() -{ - return fPerformanceList; -} - - -BString -ColorStepView::CreateFrequencyString(uint16 frequency) -{ - BString string = ""; - if (frequency >= 1000) { - char buffer [10]; - sprintf (buffer, "%.1f", float(frequency) / 1000); - string << buffer; - string += " GHz"; - } - else { - string << frequency; - string += " MHz"; - } - return string; -} - - - -float -ColorStepView::UsageOfStep(int32 step, int32 nSteps, float base) -{ - float singleWidth = (1 - base) / (nSteps - 1); - return base + singleWidth * step; -} - - -void -ColorStepView::_InitView() -{ - SetViewColor(ui_color(B_PANEL_BACKGROUND_COLOR)); - - BRect sliderFrame(Bounds()); - - fSlider = new BSlider(sliderFrame, "StepSlider", - B_TRANSLATE("Step up by CPU usage"), - new BMessage(kSteppingChanged), 0, 100); - fSlider->SetModificationMessage(new BMessage(kMSGSliderChanged)); - - fSliderPosition = 0.25 - fNSteps * 0.05; - fSlider->SetPosition(fSliderPosition); - fSlider->SetLimitLabels("0%", "100%"); - fSlider->SetHashMarks(B_HASH_MARKS_BOTTOM); - fSlider->SetHashMarkCount(5); - AddChild(fSlider); -} - - -void -ColorStepView::_CalculatePerformanceSteps() -{ - for (int i = 0; i < fNSteps; i++) { - // begin with the lowest frequency - performance_step* perfState = fPerformanceList->ItemAt(fNSteps -1 - i); - perfState->cpu_usage = _PositonStep(i); - _ColorStep(i, perfState->color); - } -} - - -float -ColorStepView::_PositonStep(int32 step) -{ - if (step >= fNSteps) - return 1.0; - - return UsageOfStep(step, fNSteps, fSliderPosition); -} - - -void -ColorStepView::_ColorStep(int32 step, rgb_color &color) -{ - color.red = fLowFreqColor.red - + (fHighFreqColor.red - fLowFreqColor.red) * step / (fNSteps - 1); - color.green = fLowFreqColor.green - + (fHighFreqColor.green - fLowFreqColor.green) * step / (fNSteps - 1); - color.blue = fLowFreqColor.blue - + (fHighFreqColor.blue - fLowFreqColor.blue) * step / (fNSteps - 1); -} - diff --git a/src/preferences/cpufrequency/ColorStepView.h b/src/preferences/cpufrequency/ColorStepView.h deleted file mode 100644 index 0c44ac5b9b..0000000000 --- a/src/preferences/cpufrequency/ColorStepView.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - -#ifndef COLORSTEPVIEW_H -#define COLORSTEPVIEW_H - -#include -#include -#include -#include -#include - -#include "DriverInterface.h" - -const uint32 kSteppingChanged = '&spc'; - -struct performance_step { - float cpu_usage; // upper limit - rgb_color color; -}; - - -typedef BObjectList PerformanceList; - - -class ColorStepView : public BControl -{ - public: - ColorStepView(BRect frame); - ~ColorStepView(); - virtual void AttachedToWindow(); - virtual void DetachedFromWindow(); - virtual void FrameResized(float w,float h); - - virtual void GetPreferredSize(float *width, float *height); - virtual void Draw(BRect updateRect); - virtual void MessageReceived(BMessage *message); - virtual void SetEnabled(bool enabled); - - void SetFrequencys(StateList *list); - PerformanceList* GetPerformanceSteps(); - static BString CreateFrequencyString(uint16 frequency); - - void SetSliderPosition(float pos) { - fSlider->SetPosition(pos); - fSliderPosition = pos; - _CalculatePerformanceSteps(); - } - float GetSliderPosition() { return fSliderPosition; } - - static float UsageOfStep(int32 step, int32 nSteps, float base); - - private: - void _InitView(); - void _CalculatePerformanceSteps(); - float _PositonStep(int32 step); - void _ColorStep(int32 step, rgb_color &color); - - BSlider* fSlider; - - float fSliderPosition; - int32 fNSteps; - rgb_color fLowFreqColor; - rgb_color fHighFreqColor; - - BView* fOffScreenView; - BBitmap* fOffScreenBitmap; - - BString fMinFrequencyLabel; - BString fMaxFrequencyLabel; - - PerformanceList* fPerformanceList; - StateList* fStateList; -}; - - -#endif diff --git a/src/preferences/cpufrequency/DriverInterface.cpp b/src/preferences/cpufrequency/DriverInterface.cpp deleted file mode 100644 index 099f7bc990..0000000000 --- a/src/preferences/cpufrequency/DriverInterface.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - -#include "DriverInterface.h" - -#include - -#include -#include -#include -#include - - -#define DRIVER_DIR "/dev/power" - -CPUFreqDriverInterface::CPUFreqDriverInterface() - : fInitOK(false), - fDriverHandler(-1), - fIsWatching(false), - fWatchingMessenger(NULL) -{ - fFrequencyStates = new StateList(20, true); - - if (_FindSpeedStepDriver(DRIVER_DIR) == B_OK) - fInitOK = true; - - status_t ret; - freq_info states[MAX_CPU_FREQUENCY_STATES]; - ret = ioctl(fDriverHandler, GET_CPU_FREQ_STATES, &states, - sizeof(freq_info) * MAX_CPU_FREQUENCY_STATES); - if (ret != B_OK) - return; - - for (int i = 0; i < MAX_CPU_FREQUENCY_STATES && states[i].frequency != 0; - i++) { - fFrequencyStates->AddItem(new freq_info(states[i])); - } -} - - -CPUFreqDriverInterface::~CPUFreqDriverInterface() -{ - StopWatching(); - delete fFrequencyStates; - - if (InitCheck() == B_OK) - close(fDriverHandler); -} - - -status_t -CPUFreqDriverInterface::InitCheck() -{ - if (fInitOK) - return B_OK; - return B_ERROR; -} - - -StateList* -CPUFreqDriverInterface::GetCpuFrequencyStates() -{ - return fFrequencyStates; -} - - -freq_info* -CPUFreqDriverInterface::GetCurrentFrequencyState() -{ - uint16 stateId = 0; - status_t ret; - ret = ioctl(fDriverHandler, GET_CURENT_CPU_FREQ_STATE, &stateId, - sizeof(uint16)); - if (ret != B_OK) - return NULL; - int32 i = 0; - while (true) { - freq_info* state = fFrequencyStates->ItemAt(i); - if (!state) - break; - i++; - - if (state->id == stateId) - return state; - } - return NULL; -} - - -int32 -CPUFreqDriverInterface::GetNumberOfFrequencyStates() -{ - return fFrequencyStates->CountItems(); -} - - -status_t -CPUFreqDriverInterface::SetFrequencyState(const freq_info* state) -{ - status_t ret; - ret = ioctl(fDriverHandler, SET_CPU_FREQ_STATE, &(state->id), - sizeof(uint16)); - return ret; -} - - -status_t -CPUFreqDriverInterface::StartWatching(BHandler* target) -{ - if (fIsWatching) - return B_ERROR; - - if (fWatchingMessenger) - delete fWatchingMessenger; - fWatchingMessenger = new BMessenger(target); - - status_t status = B_ERROR; - fThreadId = spawn_thread(&_ThreadWatchFreqFunction, "FreqThread", - B_LOW_PRIORITY, this); - if (fThreadId >= 0) - status = resume_thread(fThreadId); - else - return fThreadId; - - if (status == B_OK) { - fIsWatching = true; - return B_OK; - } - return status; -} - - -status_t -CPUFreqDriverInterface::StopWatching() -{ - - if (fIsWatching && - ioctl(fDriverHandler, STOP_WATCHING_CPU_FREQ) == B_OK) - { - status_t status; - status = wait_for_thread(fThreadId, &status); - - delete fWatchingMessenger; - fWatchingMessenger = NULL; - - fIsWatching = false; - } - - return B_ERROR; -} - - -int32 -CPUFreqDriverInterface::_ThreadWatchFreqFunction(void* data) -{ - CPUFreqDriverInterface* that = (CPUFreqDriverInterface*)data; - that->_WatchFrequency(); - return 0; -} - - -void -CPUFreqDriverInterface::_WatchFrequency() -{ - uint16 newId = 0; - while (ioctl(fDriverHandler, WATCH_CPU_FREQ, &newId, - sizeof(uint16)) == B_OK) - { - int i = 0; - while (true) { - freq_info* state = fFrequencyStates->ItemAt(i); - if (!state) - break; - i++; - - if (state->id == newId) { - BMessage msg(kMSGFrequencyChanged); - msg.AddPointer("freq_info", state); - fWatchingMessenger->SendMessage(&msg); - break; - } - } - } -} - - -status_t -CPUFreqDriverInterface::_FindSpeedStepDriver(const char* path) -{ - BDirectory dir(path); - BEntry entry; - - while (dir.GetNextEntry(&entry) == B_OK) { - BPath path; - entry.GetPath(&path); - - if (entry.IsDirectory()) { - if (_FindSpeedStepDriver(path.Path()) == B_OK) - return B_OK; - } - else { - fDriverHandler = open(path.Path(), O_RDWR); - if (fDriverHandler >= 0) { - uint32 magicId = 0; - status_t ret; - ret = ioctl(fDriverHandler, IDENTIFY_DEVICE, &magicId, - sizeof(uint32)); - if (ret == B_OK && magicId == kMagicFreqID) - return B_OK; - else - close(fDriverHandler); - } - } - - } - return B_ERROR; -} diff --git a/src/preferences/cpufrequency/DriverInterface.h b/src/preferences/cpufrequency/DriverInterface.h deleted file mode 100644 index 7a31f04de9..0000000000 --- a/src/preferences/cpufrequency/DriverInterface.h +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - -#ifndef DRIVERINTERFACE_H -#define DRIVERINTERFACE_H - -#include -#include - -#include "device/power_managment.h" - -const uint32 kMSGFrequencyChanged = '&frc'; - -typedef BObjectList StateList; - - -class CPUFreqDriverInterface -{ - public: - CPUFreqDriverInterface(); - ~CPUFreqDriverInterface(); - - status_t InitCheck(); - StateList* GetCpuFrequencyStates(); - freq_info* GetCurrentFrequencyState(); - int32 GetNumberOfFrequencyStates(); - status_t SetFrequencyState(const freq_info* state); - - status_t StartWatching(BHandler* target); - status_t StopWatching(); - private: - static int32 _ThreadWatchFreqFunction(void* data); - void _WatchFrequency(); - - thread_id fThreadId; - status_t _FindSpeedStepDriver(const char* path); - - bool fInitOK; - int32 fDriverHandler; - - StateList* fFrequencyStates; - - bool fIsWatching; - BMessenger* fWatchingMessenger; -}; - - -#endif diff --git a/src/preferences/cpufrequency/Jamfile b/src/preferences/cpufrequency/Jamfile deleted file mode 100644 index a752a37d63..0000000000 --- a/src/preferences/cpufrequency/Jamfile +++ /dev/null @@ -1,24 +0,0 @@ -SubDir HAIKU_TOP src preferences cpufrequency ; - -UsePrivateHeaders shared ; - -Preference CPUFrequency : - ColorStepView.cpp - CPUFrequencyView.cpp - DriverInterface.cpp - main.cpp - StatusView.cpp - : be $(TARGET_LIBSUPC++) localestub - : cpufrequency.rdef - ; - -DoCatalogs CPUFrequency : - x-vnd.Haiku-CPUFrequencyPref - : - ColorStepView.cpp - CPUFrequencyView.cpp - main.cpp - StatusView.cpp - PreferencesWindow.h -; - diff --git a/src/preferences/cpufrequency/PreferencesWindow.h b/src/preferences/cpufrequency/PreferencesWindow.h deleted file mode 100644 index 03e99002a8..0000000000 --- a/src/preferences/cpufrequency/PreferencesWindow.h +++ /dev/null @@ -1,481 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ -#ifndef PREFERENCES_WINDOW_h -#define PREFERENCES_WINDOW_h - - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -#if DEBUG -# define LOG(text...) PRINT((text)) -#else -# define LOG(text...) -#endif - - -#undef B_TRANSLATION_CONTEXT -#define B_TRANSLATION_CONTEXT "Pref Window" - -// messages PrefFileWatcher -const uint32 kUpdatedPreferences = '&UdP'; - - -template -class PreferencesStorage { -public: - PreferencesStorage(const char* file, - const Preferences& defaultPreferences); - ~PreferencesStorage(); - - void Revert(); - void Defaults(); - - BPoint WindowPosition() {return fWindowPosition; } - void SetWindowPosition(BPoint position) - { fWindowPosition = position; } - - Preferences* GetPreferences() { return &fPreferences; } - - status_t LoadPreferences(); - status_t SavePreferences(); - - BString& PreferencesFile() { return fPreferencesFile; } - status_t GetPreferencesPath(BPath &path); - - bool DefaultsSet(); - bool StartPrefsSet(); - -private: - BString fPreferencesFile; - - Preferences fPreferences; - Preferences fStartPreferences; - const Preferences& fDefaultPreferences; - BPoint fWindowPosition; -}; - - -template -class PrefFileWatcher : public BMessageFilter { -public: - PrefFileWatcher( - PreferencesStorage* storage, - BHandler* target); - virtual ~PrefFileWatcher(); - - virtual filter_result Filter(BMessage* message, BHandler** _target); - -private: - PreferencesStorage* fPreferencesStorage; - node_ref fPreferencesNode; - node_ref fPreferencesDirectoryNode; - - BHandler* fTarget; -}; - - -template -class PreferencesWindow : public BWindow, - public PreferencesStorage { -public: - PreferencesWindow(const char* title, - const char* file, - const Preferences& defaultPreferences); - virtual ~PreferencesWindow(); - virtual void MessageReceived(BMessage *msg); - virtual bool QuitRequested(); - - virtual bool SetPreferencesView(BView* prefView); - -private: - void _MoveToPosition(); - void _UpdateButtons(); - - BView* fPreferencesView; - BButton* fRevertButton; - BButton* fDefaultButton; - BGroupLayout* fRootLayout; -}; - - -const uint32 kDefaultMsg = 'dems'; -const uint32 kRevertMsg = 'rems'; -const uint32 kConfigChangedMsg = '&cgh'; - - -template -PreferencesStorage::PreferencesStorage(const char* file, - const Preferences& defaultPreferences) - : - fDefaultPreferences(defaultPreferences) -{ - // default center position - fWindowPosition.x = -1; - fWindowPosition.y = -1; - - fPreferencesFile = file; - if (LoadPreferences() != B_OK) - Defaults(); - fStartPreferences = fPreferences; -} - - -template -PreferencesStorage::~PreferencesStorage() -{ - SavePreferences(); -} - - -template -void -PreferencesStorage::Revert() -{ - fPreferences = fStartPreferences; -} - - -template -void -PreferencesStorage::Defaults() -{ - fPreferences = fDefaultPreferences; -} - - -template -status_t -PreferencesStorage::GetPreferencesPath(BPath &path) -{ - status_t status = find_directory(B_USER_SETTINGS_DIRECTORY, &path); - if (status < B_OK) - return status; - - return path.Append(fPreferencesFile.String()); -} - - -template -status_t -PreferencesStorage::LoadPreferences() -{ - BPath path; - status_t status = GetPreferencesPath(path); - if (status != B_OK) - return status; - - BFile settingsFile(path.Path(), B_READ_ONLY); - status = settingsFile.InitCheck(); - if (status != B_OK) - return status; - - if (settingsFile.Read(&fWindowPosition, sizeof(BPoint)) - != sizeof(BPoint)) { - LOG("failed to load settings\n"); - return B_ERROR; - } - - if (settingsFile.Read(&fPreferences, sizeof(Preferences)) - != sizeof(Preferences)) { - LOG("failed to load settings\n"); - return B_ERROR; - } - - return B_OK; -} - - -template -status_t -PreferencesStorage::SavePreferences() -{ - BPath path; - status_t status = GetPreferencesPath(path); - if (status != B_OK) - return status; - - BFile settingsFile(path.Path(), B_READ_WRITE | B_CREATE_FILE); - status = settingsFile.InitCheck(); - if (status != B_OK) { - LOG("InitCheck() settings file failed \n"); - return status; - } - - if (settingsFile.Write(&fWindowPosition, sizeof(BPoint)) - != sizeof(BPoint)) { - LOG("can't save window position\n"); - return B_ERROR; - } - - if (settingsFile.Write(&fPreferences, sizeof(Preferences)) - != sizeof(Preferences)) { - LOG("can't save settings\n"); - return B_ERROR; - } - - return B_OK; -} - - -template -bool -PreferencesStorage::DefaultsSet() -{ - return fPreferences.IsEqual(fDefaultPreferences); -} - - -template -bool -PreferencesStorage::StartPrefsSet() -{ - return fPreferences.IsEqual(fStartPreferences); -} - - -template -PrefFileWatcher::PrefFileWatcher( - PreferencesStorage* storage, BHandler* target) - : - BMessageFilter(B_PROGRAMMED_DELIVERY, B_ANY_SOURCE), - fPreferencesStorage(storage), - fTarget(target) -{ - BPath path; - find_directory(B_USER_SETTINGS_DIRECTORY, &path); - - BEntry entry(path.Path()); - if (entry.GetNodeRef(&fPreferencesDirectoryNode) == B_OK) - watch_node(&fPreferencesDirectoryNode, B_WATCH_DIRECTORY, fTarget); - - path.Append(fPreferencesStorage->PreferencesFile().String()); - entry.SetTo(path.Path()); - - if (entry.GetNodeRef(&fPreferencesNode) == B_OK) - watch_node(&fPreferencesNode, B_WATCH_STAT, fTarget); -} - - -template -PrefFileWatcher::~PrefFileWatcher() -{ - stop_watching(fTarget); -} - - -template -filter_result -PrefFileWatcher::Filter(BMessage *msg, BHandler **target) -{ - const char *name; - ino_t dir = -1; - filter_result result = B_DISPATCH_MESSAGE; - int32 opcode; - BPath path; - node_ref nref; - - if (msg->what != B_NODE_MONITOR - || msg->FindInt32("opcode", &opcode) != B_OK) - return result; - - switch (opcode) { - case B_ENTRY_MOVED: - msg->FindInt64("to directory", dir); - if (dir != fPreferencesDirectoryNode.node) - break; - // supposed to fall through - - case B_ENTRY_CREATED: - msg->FindString("name", &name); - fPreferencesStorage->GetPreferencesPath(path); - if (path.Path() == name) { - msg->FindInt32("device", &fPreferencesNode.device); - msg->FindInt64("node", &fPreferencesNode.node); - watch_node(&fPreferencesNode, B_WATCH_STAT, fTarget); - } - fPreferencesStorage->LoadPreferences(); - msg->what = kUpdatedPreferences; - break; - - case B_ENTRY_REMOVED: - msg->FindInt32("device", &nref.device); - msg->FindInt64("node", &nref.node); - if (fPreferencesNode == nref) { - // stop all watching - stop_watching(fTarget); - // and start watching the directory again - watch_node(&fPreferencesDirectoryNode, B_WATCH_DIRECTORY, - fTarget); - msg->what = kUpdatedPreferences; - } - break; - - case B_STAT_CHANGED: - msg->FindInt32("device", &nref.device); - msg->FindInt64("node", &nref.node); - if (fPreferencesNode == nref) { - fPreferencesStorage->LoadPreferences(); - msg->what = kUpdatedPreferences; - } - break; - } - return result; -} - - -template -PreferencesWindow::PreferencesWindow(const char* title, - const char* file, const Preferences& defaultPreferences) - : - BWindow(BRect(50, 50, 400, 350), title, B_TITLED_WINDOW, - B_NOT_RESIZABLE | B_NOT_ZOOMABLE | B_ASYNCHRONOUS_CONTROLS), - PreferencesStorage(file, defaultPreferences), - fPreferencesView(NULL) -{ - BGroupView* buttonView = new BGroupView(B_HORIZONTAL); - fDefaultButton = new BButton(B_TRANSLATE("Defaults"), - new BMessage(kDefaultMsg)); - buttonView->AddChild(fDefaultButton); - - fRevertButton = new BButton(B_TRANSLATE("Revert"), - new BMessage(kRevertMsg)); - buttonView->AddChild(fRevertButton); - buttonView->GetLayout()->AddItem(BSpaceLayoutItem::CreateGlue()); - - _UpdateButtons(); - - SetLayout(new BGroupLayout(B_VERTICAL)); - fRootLayout = new BGroupLayout(B_VERTICAL); - fRootLayout->SetInsets(B_USE_DEFAULT_SPACING); - fRootLayout->SetSpacing(B_USE_DEFAULT_SPACING); - BView* rootView = new BView("root view", 0, fRootLayout); - AddChild(rootView); - rootView->SetViewColor(ui_color(B_PANEL_BACKGROUND_COLOR)); - - fRootLayout->AddView(buttonView); - - BSize size = fRootLayout->PreferredSize(); - ResizeTo(size.width, size.height); - _MoveToPosition(); -} - - -template -PreferencesWindow::~PreferencesWindow() -{ - PreferencesStorage::SetWindowPosition(Frame().LeftTop()); -} - - -template -void -PreferencesWindow::MessageReceived(BMessage *msg) -{ - switch(msg->what) - { - case kConfigChangedMsg: - _UpdateButtons(); - break; - - case kDefaultMsg: - PreferencesStorage::Defaults(); - _UpdateButtons(); - if (fPreferencesView) - PostMessage(kDefaultMsg, fPreferencesView); - break; - - case kRevertMsg: - PreferencesStorage::Revert(); - _UpdateButtons(); - if (fPreferencesView) - PostMessage(kRevertMsg, fPreferencesView); - break; - - default: - BWindow::MessageReceived(msg); - } -} - - -template -bool -PreferencesWindow::QuitRequested() -{ - be_app->PostMessage(B_QUIT_REQUESTED); - return true; -} - - -template -bool -PreferencesWindow::SetPreferencesView(BView* prefView) -{ - if (fPreferencesView) - return false; - - fPreferencesView = prefView; - fRootLayout->AddView(0, fPreferencesView); - - BSize size = fRootLayout->PreferredSize(); - ResizeTo(size.width, size.height); - _MoveToPosition(); - - return true; -} - - -template -void -PreferencesWindow::_MoveToPosition() -{ - BPoint position = PreferencesStorage::WindowPosition(); - // center window on screen if it had a bad position - if (position.x < 0 && position.y < 0){ - BRect rect = BScreen().Frame(); - BRect windowFrame = Frame(); - position.x = (rect.Width() - windowFrame.Width()) / 2; - position.y = (rect.Height() - windowFrame.Height()) / 2; - } - MoveTo(position); -} - - -template -void -PreferencesWindow::_UpdateButtons() -{ - if (!PreferencesStorage::DefaultsSet()) - fDefaultButton->SetEnabled(true); - else - fDefaultButton->SetEnabled(false); - if (!PreferencesStorage::StartPrefsSet()) - fRevertButton->SetEnabled(true); - else - fRevertButton->SetEnabled(false); -} - - -#undef B_TRANSLATION_CONTEXT - -#endif // PREFERENCES_WINDOW_h diff --git a/src/preferences/cpufrequency/StatusView.cpp b/src/preferences/cpufrequency/StatusView.cpp deleted file mode 100644 index 40843a4e31..0000000000 --- a/src/preferences/cpufrequency/StatusView.cpp +++ /dev/null @@ -1,712 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - - -#include "StatusView.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "CPUFrequencyView.h" - - -extern "C" _EXPORT BView *instantiate_deskbar_item(void); - - -#undef B_TRANSLATION_CONTEXT -#define B_TRANSLATION_CONTEXT "Status view" -#define MAX_FREQ_STRING "9999MHz" - - -/* This file is used both by the preference panel and the deskbar replicant. - * This make it needs two different BCatalogs depending on the context. - * As this is not really possible with the current way BCatalog are built, - * we get the catalog by hand. This makes us unable to use the TR macros, - * and then collectcatkeys will not see the strings in the file. - * So we mark them explicitly here. - */ -B_TRANSLATE_MARK_VOID("Dynamic performance"); -B_TRANSLATE_MARK_VOID("High performance"); -B_TRANSLATE_MARK_VOID("Low energy"); -B_TRANSLATE_MARK_VOID("Set state"); -B_TRANSLATE_MARK_VOID("CPUFrequency\n" - "\twritten by Clemens Zeidler\n" - "\tCopyright 2009, Haiku, Inc.\n"); -B_TRANSLATE_MARK_VOID("Ok"); -B_TRANSLATE_MARK_VOID("Open Speedstep preferences" B_UTF8_ELLIPSIS); -B_TRANSLATE_MARK_VOID("Quit"); - -// messages FrequencySwitcher -const uint32 kMsgDynamicPolicyPulse = '&dpp'; - -// messages menu -const uint32 kMsgPolicyDynamic = 'pody'; -const uint32 kMsgPolicyPerformance = 'popr'; -const uint32 kMsgPolicyLowEnergy = 'pole'; -const uint32 kMsgPolicySetState = 'poss'; - -// messages StatusView -const uint32 kMsgOpenSSPreferences = 'ossp'; -const char* kDeskbarItemName = "CPUFreqStatusView"; - - -FrequencySwitcher::FrequencySwitcher(CPUFreqDriverInterface* interface, - BHandler* target) - : - BMessageFilter(B_PROGRAMMED_DELIVERY, B_ANY_SOURCE), - fDriverInterface(interface), - fTarget(target), - fMessageRunner(NULL), - fCurrentFrequency(NULL), - fDynamicPolicyStarted(false) -{ -} - - -FrequencySwitcher::~FrequencySwitcher() -{ - freq_preferences dummyPref; - _StartDynamicPolicy(false, dummyPref); -} - - -filter_result -FrequencySwitcher::Filter(BMessage* message, BHandler** target) -{ - filter_result result = B_DISPATCH_MESSAGE; - if (message->what == kMsgDynamicPolicyPulse) { - _CalculateDynamicState(); - result = B_SKIP_MESSAGE; - } - return result; -} - - -void -FrequencySwitcher::SetMode(const freq_preferences& pref) -{ - int16 stateCount = fDriverInterface->GetNumberOfFrequencyStates(); - StateList* list = fDriverInterface->GetCpuFrequencyStates(); - freq_info* currentState = fDriverInterface->GetCurrentFrequencyState(); - freq_info* state = NULL; - bool isDynamic = false; - - switch (pref.mode) { - case DYNAMIC: - isDynamic = true; - fSteppingThreshold = pref.stepping_threshold; - if (fMessageRunner && fIntegrationTime != pref.integration_time) { - fIntegrationTime = pref.integration_time; - fMessageRunner->SetInterval(fIntegrationTime); - } - if (!fDynamicPolicyStarted) - _StartDynamicPolicy(true, pref); - break; - - case PERFORMANCE: - state = list->ItemAt(int32(0)); - if (state != currentState) - fDriverInterface->SetFrequencyState(state); - break; - - case LOW_ENERGIE: - state = list->ItemAt(stateCount - 1); - if (state != currentState) - fDriverInterface->SetFrequencyState(state); - break; - - case CUSTOM: - if (pref.custom_stepping < stateCount) { - state = list->ItemAt(pref.custom_stepping); - fDriverInterface->SetFrequencyState(state); - } - break; - } - - if (!isDynamic && fDynamicPolicyStarted) { - fDynamicPolicyStarted = false; - _StartDynamicPolicy(false, pref); - } -} - - -void -FrequencySwitcher::_CalculateDynamicState() -{ - system_info sysInfo; - get_system_info(&sysInfo); - bigtime_t now = system_time(); - bigtime_t activeTime = sysInfo.cpu_infos[0].active_time; - - // if the dynamic mode is not started first init the prev values - if (!fDynamicPolicyStarted) { - fPrevActiveTime = activeTime; - fPrevTime = now; - fDynamicPolicyStarted = true; - } else { - float usage = (float)(activeTime - fPrevActiveTime ) - / (now - fPrevTime); - if (usage >= 1.0) - usage = 0.9999999; - - int32 numberOfStates = fDriverInterface->GetNumberOfFrequencyStates(); - for (int i = 0; i < numberOfStates; i++) { - float usageOfStep = ColorStepView::UsageOfStep(i, numberOfStates, - fSteppingThreshold); - - if (usage < usageOfStep) { - StateList* list = fDriverInterface->GetCpuFrequencyStates(); - freq_info* newState = list->ItemAt(numberOfStates - 1 - i); - if (newState != fCurrentFrequency) { - LOG("change freq\n"); - fDriverInterface->SetFrequencyState(newState); - fCurrentFrequency = newState; - } - break; - } - } - fPrevActiveTime = activeTime; - fPrevTime = now; - } -} - - -void -FrequencySwitcher::_StartDynamicPolicy(bool start, const freq_preferences& pref) -{ - if (start) { - if (!fMessageRunner) { - fIntegrationTime = pref.integration_time; - fMessageRunner = new BMessageRunner(fTarget, - new BMessage(kMsgDynamicPolicyPulse), pref.integration_time, -1); - fCurrentFrequency = fDriverInterface->GetCurrentFrequencyState(); - } - } else { - delete fMessageRunner; - fMessageRunner = NULL; - } -} - - -// #pragma mark - - - -FrequencyMenu::FrequencyMenu(BMenu* menu, BHandler* target, - PreferencesStorage* storage, - CPUFreqDriverInterface* interface) - : BMessageFilter(B_PROGRAMMED_DELIVERY, B_LOCAL_SOURCE), - fTarget(target), - fStorage(storage), - fInterface(interface) -{ - fDynamicPerformance = new BMenuItem( - B_TRANSLATE("Dynamic performance"), - new BMessage(kMsgPolicyDynamic)); - fHighPerformance = new BMenuItem( - B_TRANSLATE("High performance"), - new BMessage(kMsgPolicyPerformance)); - fLowEnergie = new BMenuItem(B_TRANSLATE("Low energy"), - new BMessage(kMsgPolicyLowEnergy)); - - menu->AddItem(fDynamicPerformance); - menu->AddItem(fHighPerformance); - menu->AddItem(fLowEnergie); - - fCustomStateMenu = new BMenu(B_TRANSLATE("Set state")); - - StateList* stateList = fInterface->GetCpuFrequencyStates(); - for (int i = 0; i < stateList->CountItems(); i++) { - freq_info* info = stateList->ItemAt(i); - BString label; - label << info->frequency; - label += " MHz"; - fCustomStateMenu->AddItem(new BMenuItem(label.String(), - new BMessage(kMsgPolicySetState))); - } - - menu->AddItem(fCustomStateMenu); - - // set the target of the items - fDynamicPerformance->SetTarget(fTarget); - fHighPerformance->SetTarget(fTarget); - fLowEnergie->SetTarget(fTarget); - - fCustomStateMenu->SetTargetForItems(fTarget); -} - - -inline void -FrequencyMenu::_SetL1MenuLabelFrom(BMenuItem* item) -{ - BMenuItem* superItem, *markedItem; - superItem = item->Menu()->Superitem(); - if (superItem) - superItem->SetLabel(item->Label()); - markedItem = item->Menu()->FindMarked(); - if (markedItem) - markedItem->SetMarked(false); - item->SetMarked(true); -} - - -filter_result -FrequencyMenu::Filter(BMessage* msg, BHandler** target) -{ - filter_result result = B_DISPATCH_MESSAGE; - - BMenuItem* item, *superItem, *markedItem; - msg->FindPointer("source", (void**)&item); - if (!item) - return result; - - bool safeChanges = false; - freq_preferences* pref = fStorage->GetPreferences(); - - switch (msg->what) { - case kMsgPolicyDynamic: - pref->mode = DYNAMIC; - _SetL1MenuLabelFrom(item); - safeChanges = true; - msg->what = kUpdatedPreferences; - break; - - case kMsgPolicyPerformance: - pref->mode = PERFORMANCE; - _SetL1MenuLabelFrom(item); - safeChanges = true; - msg->what = kUpdatedPreferences; - break; - - case kMsgPolicyLowEnergy: - pref->mode = LOW_ENERGIE; - _SetL1MenuLabelFrom(item); - safeChanges = true; - msg->what = kUpdatedPreferences; - break; - - case kMsgPolicySetState: - pref->mode = CUSTOM; - pref->custom_stepping = item->Menu()->IndexOf(item); - - superItem = item->Menu()->Supermenu()->Superitem(); - if (superItem) - superItem->SetLabel(item->Label()); - markedItem = item->Menu()->Supermenu()->FindMarked(); - if (markedItem) - markedItem->SetMarked(false); - markedItem = item->Menu()->FindMarked(); - if (markedItem) - markedItem->SetMarked(false); - item->SetMarked(true); - - safeChanges = true; - msg->what = kUpdatedPreferences; - break; - } - - if (safeChanges) - fStorage->SavePreferences(); - - return result; -} - - -void -FrequencyMenu::UpdateMenu() -{ - freq_preferences* pref = fStorage->GetPreferences(); - switch (pref->mode) { - case DYNAMIC: - _SetL1MenuLabelFrom(fDynamicPerformance); - break; - - case PERFORMANCE: - _SetL1MenuLabelFrom(fHighPerformance); - break; - - case LOW_ENERGIE: - _SetL1MenuLabelFrom(fLowEnergie); - break; - - case CUSTOM: - { - BMenuItem* markedItem = fCustomStateMenu->FindMarked(); - if (markedItem) - markedItem->SetMarked(false); - BMenuItem* customItem - = fCustomStateMenu->ItemAt(pref->custom_stepping); - if (customItem) - customItem->SetMarked(true); - BMenuItem* superItem = fCustomStateMenu->Supermenu()->Superitem(); - if (superItem && customItem) - superItem->SetLabel(customItem->Label()); - break; - } - } -} - - -// #pragma mark - - - -StatusView::StatusView(BRect frame, bool inDeskbar, - PreferencesStorage* storage) - : - BView(frame, kDeskbarItemName, B_FOLLOW_LEFT | B_FOLLOW_TOP, - B_WILL_DRAW | B_FRAME_EVENTS), - fInDeskbar(inDeskbar), - fCurrentFrequency(NULL), - fDragger(NULL) -{ - if (!inDeskbar) { - // we were obviously added to a standard window - let's add a dragger - BRect bounds = Bounds(); - bounds.top = bounds.bottom - 7; - bounds.left = bounds.right - 7; - fDragger = new BDragger(bounds, this, - B_FOLLOW_NONE); - AddChild(fDragger); - } - - if (storage) { - fOwningStorage = false; - fStorage = storage; - } else { - fOwningStorage = true; - fStorage = new PreferencesStorage("CPUFrequency", - kDefaultPreferences); - } - - _Init(); -} - - -StatusView::StatusView(BMessage* archive) - : BView(archive), - fInDeskbar(false), - fCurrentFrequency(NULL), - fDragger(NULL) -{ - app_info info; - if (be_app->GetAppInfo(&info) == B_OK - && !strcasecmp(info.signature, "application/x-vnd.Be-TSKB")) - fInDeskbar = true; - - fOwningStorage = true; - fStorage = new PreferencesStorage(kPreferencesFileName, - kDefaultPreferences); - _Init(); -} - - -StatusView::~StatusView() -{ - if (fOwningStorage) - delete fStorage; -} - - -void -StatusView::_AboutRequested() -{ - BAlert *alert = new BAlert("about", B_TRANSLATE("CPUFrequency\n" - "\twritten by Clemens Zeidler\n" - "\tCopyright 2009, Haiku, Inc.\n"), - B_TRANSLATE("OK")); - BTextView *view = alert->TextView(); - BFont font; - - view->SetStylable(true); - - view->GetFont(&font); - font.SetSize(18); - font.SetFace(B_BOLD_FACE); - view->SetFontAndColor(0, 13, &font); - - alert->SetFlags(alert->Flags() | B_CLOSE_ON_ESCAPE); - alert->Go(); -} - - -void -StatusView::_Quit() -{ - if (fInDeskbar) { - BDeskbar deskbar; - deskbar.RemoveItem(kDeskbarItemName); - } else - be_app->PostMessage(B_QUIT_REQUESTED); -} - - -StatusView* -StatusView::Instantiate(BMessage* archive) -{ - if (!validate_instantiation(archive, "StatusView")) - return NULL; - - return new StatusView(archive); -} - - -status_t -StatusView::Archive(BMessage* archive, bool deep) const -{ - status_t status = BView::Archive(archive, deep); - if (status == B_OK) - status = archive->AddString("add_on", kPrefSignature); - if (status == B_OK) - status = archive->AddString("class", "StatusView"); - - return status; -} - - -void -StatusView::AttachedToWindow() -{ - BView::AttachedToWindow(); - if (Parent()) { - if ((Parent()->Flags() & B_DRAW_ON_CHILDREN) != 0) - SetViewColor(B_TRANSPARENT_COLOR); - else - SetViewColor(Parent()->ViewColor()); - } else - SetViewColor(ui_color(B_PANEL_BACKGROUND_COLOR)); - - SetLowColor(ViewColor()); - - // watching if the driver change the frequency - fDriverInterface.StartWatching(this); - - // monitor preferences file - fPrefFileWatcher = new PrefFileWatcher(fStorage, this); - AddFilter(fPrefFileWatcher); - - // FrequencySwitcher - fFrequencySwitcher = new FrequencySwitcher(&fDriverInterface, this); - fFrequencySwitcher->SetMode(*(fStorage->GetPreferences())); - AddFilter(fFrequencySwitcher); - - // perferences menu - fPreferencesMenu = new BPopUpMenu(B_EMPTY_STRING, false, false); - fPreferencesMenuFilter = new FrequencyMenu(fPreferencesMenu, this, - fStorage, &fDriverInterface); - - fPreferencesMenu->SetFont(be_plain_font); - - fPreferencesMenu->AddSeparatorItem(); - fOpenPrefItem = new BMenuItem(B_TRANSLATE( - "Open Speedstep preferences" B_UTF8_ELLIPSIS), - new BMessage(kMsgOpenSSPreferences)); - fPreferencesMenu->AddItem(fOpenPrefItem); - fOpenPrefItem->SetTarget(this); - - if (fInDeskbar) { - fQuitItem= new BMenuItem(B_TRANSLATE("Quit"), - new BMessage(B_QUIT_REQUESTED)); - fPreferencesMenu->AddItem(fQuitItem); - fQuitItem->SetTarget(this); - } - AddFilter(fPreferencesMenuFilter); - - fPreferencesMenuFilter->UpdateMenu(); -} - - -void -StatusView::DetachedFromWindow() -{ - fDriverInterface.StopWatching(); - - if (RemoveFilter(fPrefFileWatcher)) - delete fPrefFileWatcher; - if (RemoveFilter(fFrequencySwitcher)) - delete fFrequencySwitcher; - if (RemoveFilter(fPreferencesMenuFilter)) - delete fPreferencesMenuFilter; - delete fPreferencesMenu; -} - - -void -StatusView::MessageReceived(BMessage* message) -{ - switch (message->what) { - case kMsgOpenSSPreferences: - _OpenPreferences(); - break; - - case kMSGFrequencyChanged: - message->FindPointer("freq_info", (void**)&fCurrentFrequency); - _SetupNewFreqString(); - Invalidate(); - break; - - case kUpdatedPreferences: - fFrequencySwitcher->SetMode(*(fStorage->GetPreferences())); - fPreferencesMenuFilter->UpdateMenu(); - break; - - case B_ABOUT_REQUESTED: - _AboutRequested(); - break; - - case B_QUIT_REQUESTED: - _Quit(); - break; - - default: - BView::MessageReceived(message); - } -} - - -void -StatusView::FrameResized(float width, float height) -{ - Invalidate(); -} - - -void -StatusView::Draw(BRect updateRect) -{ - font_height fontHeight; - GetFontHeight(&fontHeight); - float height = fontHeight.ascent + fontHeight.descent; - - if (be_control_look != NULL) { - be_control_look->DrawLabel(this, fFreqString.String(), - Parent()->ViewColor(), 0, BPoint(0, height)); - } else { - MovePenTo(0, height); - DrawString(fFreqString.String()); - } -} - - - - -void -StatusView::MouseDown(BPoint point) -{ - if (fShowPopUpMenu) { - ConvertToScreen(&point); - fPreferencesMenu->Go(point, true, false, true); - } -} - - -void -StatusView::GetPreferredSize(float *width, float *height) -{ - font_height fontHeight; - GetFontHeight(&fontHeight); - *height = fontHeight.ascent + fontHeight.descent; - if (!fInDeskbar) - *height += 7; - - *width = StringWidth(MAX_FREQ_STRING); -} - - -void -StatusView::ResizeToPreferred(void) -{ - float width, height; - GetPreferredSize(&width, &height); - ResizeTo(width, height); -} - - -void -StatusView::ShowPopUpMenu(bool show) -{ - fShowPopUpMenu = show; -} - - -void -StatusView::UpdateCPUFreqState() -{ - fFrequencySwitcher->SetMode(*(fStorage->GetPreferences())); -} - - -void -StatusView::_Init() -{ - fShowPopUpMenu = true; - fCurrentFrequency = fDriverInterface.GetCurrentFrequencyState(); - - _SetupNewFreqString(); -} - - -void -StatusView::_SetupNewFreqString() -{ - if (fCurrentFrequency) { - fFreqString = ColorStepView::CreateFrequencyString( - fCurrentFrequency->frequency); - } else - fFreqString = "? MHz"; - - ResizeToPreferred(); - - if (fDragger) { - BRect bounds = Bounds(); - fDragger->MoveTo(bounds.right - 7, bounds.bottom - 7); - } -} - -void -StatusView::_OpenPreferences() -{ - status_t ret = be_roster->Launch(kPrefSignature); - if (ret == B_ALREADY_RUNNING) { - app_info info; - ret = be_roster->GetAppInfo(kPrefSignature, &info); - if (ret == B_OK) - ret = be_roster->ActivateApp(info.team); - } - if (ret < B_OK) { - BString errorMessage(B_TRANSLATE( - "Launching the CPU frequency preflet failed.\n\nError: ")); - errorMessage << strerror(ret); - BAlert* alert = new BAlert("launch error", errorMessage.String(), - "OK"); - // asynchronous alert in order to not block replicant host - // application - alert->SetFlags(alert->Flags() | B_CLOSE_ON_ESCAPE); - alert->Go(NULL); - } -} - - -// #pragma mark - - - -extern "C" _EXPORT BView* -instantiate_deskbar_item(void) -{ - return new StatusView(BRect(0, 0, 15, 15), true, NULL); -} diff --git a/src/preferences/cpufrequency/StatusView.h b/src/preferences/cpufrequency/StatusView.h deleted file mode 100644 index cc677c182b..0000000000 --- a/src/preferences/cpufrequency/StatusView.h +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ -#ifndef STATUS_VIEW_H -#define STATUS_VIEW_H - - -#include -#include -#include -#include -#include - -#include "CPUFrequencyView.h" -#include "DriverInterface.h" -#include "PreferencesWindow.h" - - -extern const char* kDeskbarItemName; -extern const char* kAddonSignature; - - -class FrequencySwitcher : public BMessageFilter { -public: - FrequencySwitcher( - CPUFreqDriverInterface* interface, - BHandler* target); - virtual ~FrequencySwitcher(); - - virtual filter_result Filter(BMessage* message, BHandler** _target); - - void SetMode(const freq_preferences& pref); - -private: - void _CalculateDynamicState(); - void _StartDynamicPolicy(bool start, - const freq_preferences& pref); - - CPUFreqDriverInterface* fDriverInterface; - BHandler* fTarget; - BMessageRunner* fMessageRunner; - - freq_info* fCurrentFrequency; - - bool fDynamicPolicyStarted; - bigtime_t fPrevActiveTime; - bigtime_t fPrevTime; - float fSteppingThreshold; - bigtime_t fIntegrationTime; -}; - - -class FrequencyMenu : public BMessageFilter { -public: - FrequencyMenu(BMenu* menu, BHandler* target, - PreferencesStorage* - storage, - CPUFreqDriverInterface* interface); - virtual filter_result Filter(BMessage* message, BHandler** _target); - - void UpdateMenu(); - -private: - inline void _SetL1MenuLabelFrom(BMenuItem* item); - - BHandler* fTarget; - BMenuItem* fDynamicPerformance; - BMenuItem* fHighPerformance; - BMenuItem* fLowEnergie; - BMenu* fCustomStateMenu; - - PreferencesStorage* fStorage; - CPUFreqDriverInterface* fInterface; -}; - - -class StatusView : public BView { -public: - StatusView(BRect frame, bool inDeskbar = false, - PreferencesStorage* - storage = NULL); - StatusView(BMessage* archive); - virtual ~StatusView(); - - static StatusView* Instantiate(BMessage* archive); - virtual status_t Archive(BMessage* archive, - bool deep = true) const; - - virtual void AttachedToWindow(); - virtual void DetachedFromWindow(); - - virtual void MessageReceived(BMessage* message); - virtual void FrameResized(float width, float height); - virtual void MouseDown(BPoint where); - virtual void Draw(BRect updateRect); - - virtual void GetPreferredSize(float *width, float *height); - virtual void ResizeToPreferred(void); - - virtual void ShowPopUpMenu(bool show = true); - - virtual void UpdateCPUFreqState(); - -private: - void _Init(); - void _SetupNewFreqString(); - void _OpenPreferences(); - void _AboutRequested(); - void _Quit(); - - bool fInDeskbar; - - CPUFreqDriverInterface fDriverInterface; - freq_info* fCurrentFrequency; - FrequencySwitcher* fFrequencySwitcher; - - bool fShowPopUpMenu; - BPopUpMenu* fPreferencesMenu; - BMenuItem* fOpenPrefItem; - BMenuItem* fQuitItem; - - FrequencyMenu* fPreferencesMenuFilter; - bool fOwningStorage; - PreferencesStorage* fStorage; - PrefFileWatcher* fPrefFileWatcher; - - BString fFreqString; - BDragger* fDragger; -}; - -#endif // STATUS_VIEW_H diff --git a/src/preferences/cpufrequency/cpufrequency.rdef b/src/preferences/cpufrequency/cpufrequency.rdef deleted file mode 100644 index 2aa9100ae6..0000000000 --- a/src/preferences/cpufrequency/cpufrequency.rdef +++ /dev/null @@ -1,45 +0,0 @@ - -resource app_signature "application/x-vnd.Haiku-CPUFrequencyPref"; - -resource app_name_catalog_entry "x-vnd.Haiku-CPUFrequencyPref:System name:CPUFrequency"; - -resource app_version { - major = 1, - middle = 0, - minor = 0, - - variety = B_APPV_FINAL, - internal = 0, - - short_info = "CPUFrequency", - long_info = "CPUFrequency ©2008-2009 Haiku, Inc." -}; - -resource app_flags B_SINGLE_LAUNCH; - -resource vector_icon { - $"6E6369660E03010000020002023980000000000000004000004BE00008908100" - $"010000FFFF01000000020016023CC7EE389BC0BA16573E39B04977C842ADC700" - $"FFFFD3020006023C529D3753A2B8966F3D9D084B6044496AAF00474747FFA5A0" - $"A002001602BC4E76BC411B3C90DABCA00D47587D4ABA850090FFD40200160238" - $"313C3B5CF0BFCD963C7AAC4C13943FCAF901ECFFC3054B04017E020006033E2F" - $"99387F17BA42DB3FF5B94A0E32482C90001D1E2C3D454658FF01010102000602" - $"3879063B8224BE2CC83B10DB4A1F6F49B894FF9A9A9A00242222020006033C69" - $"A60000000000003E186148800049800058F3F3F300D4CECEFFD9D9D902000603" - $"3C1F1A33E78CB7ACC03FFE4F48BB3EBD7B6C0078D905818CFF05FF7ADD050200" - $"1602349C2E37B5FABA1F6036FC624A3E004B320001D3FF910200160235777837" - $"0A67B7E8CE363A844A1D684B45D800F3FF2E0D0A04486050605C51544E04033E" - $"5349594856475C49604B5C4E604B0A06262A264C485E5252523030240A04262A" - $"4838523030240A044838485E525252300A04262A264C485E48380A04453A4553" - $"2844282E0A04B6F9C0F42845282EB701B8EC0A044550455328452AC0F30A0445" - $"3A45502A43B701B8EC0408AEBAB6BCBCC32FBD4F2E3930BDA8B9ACC0A5BA9EBC" - $"03BB1EBFD937BF0DBD4FC072BCC3C019BDDBC0CB46460204BF23C726BF91C70D" - $"BEB5C73FBE9FC87EBE7AC7D9BEC5C922BFAAC97BBF3CC994C018C962C02DC823" - $"C053C8C8C008C77F0204BFCEC6FBC042C6E0BF5BC715BF48C85ABF1FC7B3BF71" - $"C902C063C95ABFF0C974C0D7C93FC0EAC7FAC113C8A2C0C1C7530E0A07010000" - $"0A0101011001178300040A0001021001178400040A020103000A080109000A0B" - $"010A1001178120040A030104000A04020506000A090107000A0A0108000A0D01" - $"0C0815FF0A0C010B0815FF0A0D010C0A3FEAF70000000000003FEAF7C573B4C2" - $"770615FF0A0C010B0A3FEAF70000000000003FEAF7C573B4C2770615FF" -}; - diff --git a/src/preferences/cpufrequency/main.cpp b/src/preferences/cpufrequency/main.cpp deleted file mode 100644 index f24d576241..0000000000 --- a/src/preferences/cpufrequency/main.cpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2009, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Clemens Zeidler, haiku@clemens-zeidler.de - */ - - -#include -#include -#include - -#include "CPUFrequencyView.h" -#include "PreferencesWindow.h" - - -#undef B_TRANSLATION_CONTEXT -#define B_TRANSLATION_CONTEXT "Main window" - - -int -main(int argc, char* argv[]) -{ - BApplication *app = new BApplication(kPrefSignature); - - PreferencesWindow *window; - window = new PreferencesWindow( - B_TRANSLATE_SYSTEM_NAME("CPUFrequency"), kPreferencesFileName, - kDefaultPreferences); - CPUFrequencyView* prefView - = new CPUFrequencyView(BRect(0, 0, 400, 350), window); - window->SetPreferencesView(prefView); - window->Show(); - app->Run(); - - delete app; - return 0; -} From a04b7d9f96772b54819061e3716f1e273526a84c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 22 Nov 2013 05:30:14 +0100 Subject: [PATCH 140/273] power: Remove old cpufreq interface --- headers/private/device/power_managment.h | 29 ------------------------ 1 file changed, 29 deletions(-) diff --git a/headers/private/device/power_managment.h b/headers/private/device/power_managment.h index 1fad929c98..7964a27240 100644 --- a/headers/private/device/power_managment.h +++ b/headers/private/device/power_managment.h @@ -14,19 +14,6 @@ enum { // ioctl response with kMagicFreqID IDENTIFY_DEVICE = B_DEVICE_OP_CODES_END + 20001, - // CPU Frequence: - // get a list of freq_info, the list is terminated with a element with - // frequency = 0 - GET_CPU_FREQ_STATES = B_DEVICE_OP_CODES_END + 20005, - // get and set a freq_info - GET_CURENT_CPU_FREQ_STATE, - SET_CPU_FREQ_STATE, - // start watching for frequency changes, ioctl blocks until the frequency - // has changed - WATCH_CPU_FREQ, - // stop all watching ioctl, ioctl return B_ERROR - STOP_WATCHING_CPU_FREQ, - GET_BATTERY_INFO, GET_EXTENDED_BATTERY_INFO, WATCH_BATTERY, @@ -34,22 +21,6 @@ enum { }; -// CPU Frequence: -// magic id returned by IDENTIFY_DEVICE -const uint32 kMagicFreqID = 48921; - - -#define MAX_CPU_FREQUENCY_STATES 10 - - -typedef struct { - uint16 frequency; // [Mhz] - uint16 volts; - uint16 id; - int power; -} freq_info; - - // ACPI Battery: // magic id returned by IDENTIFY_DEVICE const uint32 kMagicACPIBatteryID = 17822; From 03f7d3d1db10c0a467bea944fb3230ee70c6e015 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 24 Nov 2013 22:51:07 +0100 Subject: [PATCH 141/273] kernel: Restore logical processor disabling --- headers/private/kernel/kscheduler.h | 2 + headers/private/kernel/thread.h | 2 + src/system/kernel/cpu.cpp | 40 +++-- src/system/kernel/int.cpp | 24 ++- src/system/kernel/scheduler/scheduler.cpp | 184 ++++++++++++++++++++-- src/system/kernel/thread.cpp | 15 +- 6 files changed, 234 insertions(+), 33 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index 92a82e9ccc..2d8222ebed 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -83,6 +83,8 @@ status_t scheduler_set_operation_mode(scheduler_mode mode); */ void scheduler_dump_thread_data(Thread* thread); +void scheduler_set_cpu_enabled(int32 cpu, bool enabled); + void scheduler_add_listener(struct SchedulerListener* listener); void scheduler_remove_listener(struct SchedulerListener* listener); diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 72d123b503..61c66387d2 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -87,6 +87,8 @@ status_t thread_preboot_init_percpu(struct kernel_args *args, int32 cpuNum); void thread_yield(void); void thread_exit(void); +void thread_map(void (*function)(Thread* thread, void* data), void* data); + int32 thread_max_threads(void); int32 thread_used_threads(void); diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 433d567c75..5ab8d26294 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -273,7 +274,6 @@ _user_cpu_enabled(int32 cpu) status_t _user_set_cpu_enabled(int32 cpu, bool enabled) { - status_t status = B_OK; cpu_status state; int32 i, count; @@ -283,8 +283,7 @@ _user_set_cpu_enabled(int32 cpu, bool enabled) // We need to lock here to make sure that no one can disable // the last CPU - state = disable_interrupts(); - acquire_spinlock(&sSetCpuLock); + InterruptsSpinLocker locker(sSetCpuLock); if (!enabled) { // check if this is the last CPU to be disabled @@ -294,14 +293,37 @@ _user_set_cpu_enabled(int32 cpu, bool enabled) } if (count == 1) - status = B_NOT_ALLOWED; + return B_NOT_ALLOWED; } - if (status == B_OK) - gCPU[cpu].disabled = !enabled; + bool oldState = gCPU[cpu].disabled; - release_spinlock(&sSetCpuLock); - restore_interrupts(state); - return status; + if (oldState != !enabled) + scheduler_set_cpu_enabled(cpu, enabled); + + if (!enabled) { + if (smp_get_current_cpu() == cpu) { + locker.Unlock(); + thread_yield(); + locker.Lock(); + } + + // someone reenabled the CPU while we were rescheduling + if (!gCPU[cpu].disabled) + return B_OK; + + ASSERT(smp_get_current_cpu() != cpu); + while (!thread_is_idle_thread(gCPU[cpu].running_thread)) { + locker.Unlock(); + thread_yield(); + locker.Lock(); + + if (!gCPU[cpu].disabled) + return B_OK; + ASSERT(smp_get_current_cpu() != cpu); + } + } + + return B_OK; } diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index baed8fd73d..5a0bebfd9d 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -391,14 +391,17 @@ restore_interrupts(cpu_status status) static uint32 assign_cpu(void) { - int32 nextID = atomic_add(&sLastCPU, 1); - cpu_topology_node* node = get_cpu_topology(); + cpu_topology_node* node; + do { + int32 nextID = atomic_add(&sLastCPU, 1); + node = get_cpu_topology(); - while (node->level != CPU_TOPOLOGY_SMT) { - int levelSize = node->children_count; - node = node->children[nextID % levelSize]; - nextID /= levelSize; - } + while (node->level != CPU_TOPOLOGY_SMT) { + int levelSize = node->children_count; + node = node->children[nextID % levelSize]; + nextID /= levelSize; + } + } while (gCPU[node->id].disabled); return node->id; } @@ -680,6 +683,12 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) int32 oldCPU = sVectors[vector].assigned_cpu.cpu; + if (newCPU == -1) + newCPU = assign_cpu(); + dprintf_no_syslog("IRQ %ld CPU %" B_PRId32 " -> CPU %" B_PRId32 "\n", vector, oldCPU, newCPU); + if (newCPU == oldCPU) + return; + ASSERT(oldCPU != -1); cpu_ent* cpu = &gCPU[oldCPU]; @@ -693,6 +702,5 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) sVectors[vector].assigned_cpu.cpu = newCPU; arch_int_assign_to_cpu(vector, newCPU); list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); - locker.Unlock(); } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 105d7fa13e..bc4fd947fb 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -399,6 +399,11 @@ update_load_heaps(int32 core) CoreEntry* entry = &gCoreEntries[core]; + if (entry->fCPUCount == 0) { + entry->fLoad = 0; + return; + } + WriteSpinLocker coreLocker(gCoreHeapsLock); int32 newKey = get_core_load(entry); @@ -626,12 +631,9 @@ thread_goes_away(Thread* thread) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - int32 core = schedulerThreadData->previous_core; - schedulerThreadData->went_sleep = system_time(); schedulerThreadData->went_sleep_active - = atomic_get64(&gCoreEntries[core].fActiveTime); + = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); } @@ -1028,7 +1030,7 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) atomic_add64(&gCoreEntries[thisCore].fActiveTime, active); } - if (!gSingleCore) + if (!gSingleCore && !gCPU[smp_get_current_cpu()].disabled) compute_cpu_load(smp_get_current_cpu()); int32 oldPriority = get_effective_priority(oldThread); @@ -1150,9 +1152,24 @@ _scheduler_reschedule(void) schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority and enqueue back the old thread - Thread* nextThread - = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, - putOldThreadAtBack); + Thread* nextThread; + if (gCPU[thisCPU].disabled) { + if (!thread_is_idle_thread(oldThread)) { + SpinLocker runQueueLocker(gCoreEntries[thisCore].fQueueLock); + + nextThread = gPinnedRunQueues[thisCPU].GetHead(B_IDLE_PRIORITY); + gPinnedRunQueues[thisCPU].Remove(nextThread); + nextThread->scheduler_data->enqueued = false; + + putOldThreadAtBack = oldThread->pinned_to_cpu == 0; + } else + nextThread = oldThread; + } else { + nextThread + = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, + putOldThreadAtBack); + } + if (nextThread != oldThread) { if (enqueueOldThread) { if (putOldThreadAtBack) @@ -1174,7 +1191,8 @@ _scheduler_reschedule(void) oldThread, nextThread); // update CPU heap - update_cpu_priority(thisCPU, get_effective_priority(nextThread)); + if (!gCPU[thisCPU].disabled) + update_cpu_priority(thisCPU, get_effective_priority(nextThread)); nextThread->state = B_THREAD_RUNNING; nextThread->next_state = B_THREAD_READY; @@ -1281,6 +1299,22 @@ scheduler_start(void) } +static inline void +acquire_big_scheduler_lock(void) +{ + for (int32_t i = 0; i < smp_get_num_cpus(); i++) + acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); +} + + +static inline void +release_big_scheduler_lock(void) +{ + for (int32_t i = 0; i < smp_get_num_cpus(); i++) + release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); +} + + status_t scheduler_set_operation_mode(scheduler_mode mode) { @@ -1292,20 +1326,144 @@ scheduler_set_operation_mode(scheduler_mode mode) dprintf("scheduler: switching to %s mode\n", sSchedulerModes[mode]->name); InterruptsLocker _; - for (int32_t i = 0; i < smp_get_num_cpus(); i++) - acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + acquire_big_scheduler_lock(); sCurrentModeID = mode; sCurrentMode = sSchedulerModes[mode]; sCurrentMode->switch_to_mode(); - for (int32_t i = 0; i < smp_get_num_cpus(); i++) - release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + release_big_scheduler_lock(); return B_OK; } +static void +unassign_thread(Thread* thread, void* data) +{ + int32 core = *(int32*)data; + + if (thread->scheduler_data->previous_core == core + && thread->pinned_to_cpu == 0) { + thread->scheduler_data->previous_core = -1; + } +} + + +void +scheduler_set_cpu_enabled(int32 cpu, bool enabled) +{ + dprintf("scheduler: %s CPU %" B_PRId32 "\n", + enabled ? "enabling" : "disabling", cpu); + + InterruptsLocker _; + acquire_big_scheduler_lock(); + + gCPU[cpu].disabled = !enabled; + + CoreEntry* core = &gCoreEntries[gCPUToCore[cpu]]; + PackageEntry* package = &gPackageEntries[gCPUToPackage[cpu]]; + + int32 oldCPUCount = core->fCPUCount; + ASSERT(oldCPUCount >= 0); + if (enabled) + core->fCPUCount++; + else { + update_cpu_priority(cpu, B_IDLE_PRIORITY); + core->fCPUCount--; + } + + if (core->fCPUCount == 0) { + // core has been disabled + ASSERT(!enabled); + + int32 load = CoreLoadHeap::GetKey(core); + if (load > kHighLoad) { + gCoreHighLoadHeap->ModifyKey(core, -1); + ASSERT(gCoreHighLoadHeap->PeekMinimum() == core); + gCoreHighLoadHeap->RemoveMinimum(); + } else { + gCoreLoadHeap->ModifyKey(core, -1); + ASSERT(gCoreLoadHeap->PeekMinimum() == core); + gCoreLoadHeap->RemoveMinimum(); + } + + package->fIdleCores.Remove(core); + package->fIdleCoreCount--; + package->fCoreCount--; + + if (package->fCoreCount == 0) + gIdlePackageList->Remove(package); + + // get rid of threads + thread_map(unassign_thread, &core->fCoreID); + + while (gRunQueues[core->fCoreID].PeekMaximum() != NULL) { + Thread* thread = gRunQueues[core->fCoreID].PeekMaximum(); + gRunQueues[core->fCoreID].Remove(thread); + thread->scheduler_data->enqueued = false; + + ASSERT(thread->scheduler_data->previous_core == -1); + enqueue(thread, false); + } + } else if (oldCPUCount == 0) { + // core has been reenabled + ASSERT(enabled); + + gCPUEntries[cpu].fLoad = 0; + core->fLoad = 0; + gCoreLoadHeap->Insert(core, 0); + + package->fCoreCount++; + package->fIdleCoreCount++; + package->fIdleCores.Add(core); + + if (package->fCoreCount == 1) + gIdlePackageList->Add(package); + } + + if (enabled) { + gCPUPriorityHeaps[core->fCoreID].Insert(&gCPUEntries[cpu], + B_IDLE_PRIORITY); + gCPUEntries[cpu].fLoad = 0; + } else { + gCPUPriorityHeaps[core->fCoreID].ModifyKey(&gCPUEntries[cpu], + THREAD_MAX_SET_PRIORITY + 1); + ASSERT(gCPUPriorityHeaps[core->fCoreID].PeekMaximum() + == &gCPUEntries[cpu]); + gCPUPriorityHeaps[core->fCoreID].RemoveMaximum(); + + core->fLoad -= gCPUEntries[cpu].fLoad; + } + + if (!enabled) { + cpu_ent* entry = &gCPU[cpu]; + + // get rid of irqs + SpinLocker locker(entry->irqs_lock); + irq_assignment* irq + = (irq_assignment*)list_get_first_item(&entry->irqs); + while (irq != NULL) { + locker.Unlock(); + + assign_io_interrupt_to_cpu(irq->irq, -1); + + locker.Lock(); + irq = (irq_assignment*)list_get_first_item(&entry->irqs); + } + locker.Unlock(); + + // don't wait until the thread quantum ends + if (smp_get_current_cpu() != cpu) { + smp_send_ici(cpu, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, + SMP_MSG_FLAG_ASYNC); + } + } + + release_big_scheduler_lock(); +} + + static void traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) { diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 3aee21c3dd..ca66a46140 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2408,8 +2408,6 @@ peek_next_thread_id() void thread_yield(void) { - // Yielding is for being nice, not for making things work. -#if !KDEBUG Thread *thread = thread_get_current_thread(); if (thread == NULL) return; @@ -2418,7 +2416,18 @@ thread_yield(void) thread->has_yielded = true; scheduler_reschedule(); -#endif +} + + +void +thread_map(void (*function)(Thread* thread, void* data), void* data) +{ + InterruptsSpinLocker threadHashLocker(sThreadHashLock); + + for (ThreadHashTable::Iterator it = sThreadHash.GetIterator(); + Thread* thread = it.Next();) { + function(thread, data); + } } From cec16c2dcfb0bddb0d9dc11fb63793c4ca9a53e0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 24 Nov 2013 22:54:14 +0100 Subject: [PATCH 142/273] spinlock: Fix panic messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks Jérôme for pointing this out. --- src/system/kernel/smp.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 08203ef201..a07df3f290 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -532,13 +532,13 @@ try_acquire_write_spinlock(rw_spinlock* lock) { #if DEBUG_SPINLOCKS if (are_interrupts_enabled()) { - panic("try_acquire_write_spinlock: attempt to acquire lock %p with" - " interrupts enabled", lock); + panic("try_acquire_write_spinlock: attempt to acquire lock %p with " + "interrupts enabled", lock); } if (sNumCPUs < 2 && lock->lock != 0) { - panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " - "non-SMP system", lock); + panic("try_acquire_write_spinlock(): attempt to acquire lock %p twice " + "on non-SMP system", lock); } #endif @@ -551,8 +551,8 @@ acquire_write_spinlock(rw_spinlock* lock) { #if DEBUG_SPINLOCKS if (are_interrupts_enabled()) { - panic("acquire_write_spinlock: attempt to acquire lock %p with" - " interrupts enabled", lock); + panic("acquire_write_spinlock: attempt to acquire lock %p with " + "interrupts enabled", lock); } #endif @@ -582,8 +582,8 @@ release_write_spinlock(rw_spinlock* lock) #if DEBUG_SPINLOCKS uint32 previous = atomic_get_and_set(&lock->lock, 0); if ((previous & 1u << 31) == 0) { - panic("release_write_spinlock: lock %p was already released (value:" - " %x)\n", lock, previous); + panic("release_write_spinlock: lock %p was already released (value: " + "%x)\n", lock, previous); } #else atomic_set(&lock->lock, 0); @@ -596,13 +596,13 @@ try_acquire_read_spinlock(rw_spinlock* lock) { #if DEBUG_SPINLOCKS if (are_interrupts_enabled()) { - panic("try_acquire_read_spinlock: attempt to acquire lock %p with" - " interrupts enabled", lock); + panic("try_acquire_read_spinlock: attempt to acquire lock %p with " + "interrupts enabled", lock); } if (sNumCPUs < 2 && lock->lock != 0) { - panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " - "non-SMP system", lock); + panic("try_acquire_read_spinlock(): attempt to acquire lock %p twice " + "on non-SMP system", lock); } #endif @@ -620,8 +620,8 @@ acquire_read_spinlock(rw_spinlock* lock) { #if DEBUG_SPINLOCKS if (are_interrupts_enabled()) { - panic("acquire_read_spinlock: attempt to acquire lock %p with" - " interrupts enabled", lock); + panic("acquire_read_spinlock: attempt to acquire lock %p with " + "interrupts enabled", lock); } #endif From f95b6fdfc84784fd24255fff8a492b877a26cbfd Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 24 Nov 2013 23:18:58 +0100 Subject: [PATCH 143/273] scheudler: Do not include irq load in thread load --- src/system/kernel/scheduler/scheduler.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index bc4fd947fb..00b1d57fed 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -617,6 +617,12 @@ compute_cpu_load(int32 cpu) static inline void compute_thread_load(Thread* thread) { + if (thread->scheduler_data->last_interrupt_time > 0) { + bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; + interruptTime -= thread->scheduler_data->last_interrupt_time; + thread->scheduler_data->measure_active_time -= interruptTime; + } + compute_load(thread->scheduler_data->measure_time, thread->scheduler_data->measure_active_time, thread->scheduler_data->load); @@ -631,6 +637,8 @@ thread_goes_away(Thread* thread) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + schedulerThreadData->last_interrupt_time = 0; + schedulerThreadData->went_sleep = system_time(); schedulerThreadData->went_sleep_active = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); From 13a89839fc09fea7d328568ddb76d8369ce2839d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 24 Nov 2013 23:20:42 +0100 Subject: [PATCH 144/273] scheduler: Fix power saving mode and other minor improvements --- src/system/kernel/int.cpp | 2 +- src/system/kernel/scheduler/low_latency.cpp | 7 ++++++ src/system/kernel/scheduler/power_saving.cpp | 23 +++++++++++++------ src/system/kernel/scheduler/scheduler.cpp | 14 +++++------ src/system/kernel/scheduler/scheduler_modes.h | 1 + 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 5a0bebfd9d..9fd6d2e884 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -685,7 +685,7 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) if (newCPU == -1) newCPU = assign_cpu(); - dprintf_no_syslog("IRQ %ld CPU %" B_PRId32 " -> CPU %" B_PRId32 "\n", vector, oldCPU, newCPU); + if (newCPU == oldCPU) return; diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 5f1224c45e..832b7ab1d3 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -19,6 +19,12 @@ switch_to_mode(void) } +static void +set_cpu_enabled(int32 /* cpu */, bool /* enabled */) +{ +} + + static bool has_cache_expired(Thread* thread) { @@ -180,6 +186,7 @@ scheduler_mode_operations gSchedulerLowLatencyMode = { true, switch_to_mode, + set_cpu_enabled, has_cache_expired, choose_core, should_rebalance, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 51485eee75..63ad0e3c23 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -16,6 +16,21 @@ using namespace Scheduler; static int32 sSmallTaskCore; +static void +switch_to_mode(void) +{ + sSmallTaskCore = -1; +} + + +static void +set_cpu_enabled(int32 cpu, bool enabled) +{ + if (!enabled) + sSmallTaskCore = -1; +} + + static bool has_cache_expired(Thread* thread) { @@ -31,13 +46,6 @@ has_cache_expired(Thread* thread) } -static void -switch_to_mode(void) -{ - sSmallTaskCore = -1; -} - - static bool try_small_task_packing(Thread* thread) { @@ -244,6 +252,7 @@ scheduler_mode_operations gSchedulerPowerSavingMode = { false, switch_to_mode, + set_cpu_enabled, has_cache_expired, choose_core, should_rebalance, diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 00b1d57fed..e1d7c45154 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -583,9 +583,6 @@ should_rebalance(Thread* thread) { ASSERT(!gSingleCore); - if (thread_is_idle_thread(thread)) - return false; - return sCurrentMode->should_rebalance(thread); } @@ -600,9 +597,6 @@ compute_cpu_load(int32 cpu) if (oldLoad < 0) return; - if (gCPUEntries[cpu].fLoad > kVeryHighLoad) - sCurrentMode->rebalance_irqs(false); - if (oldLoad != gCPUEntries[cpu].fLoad) { int32 core = gCPUToCore[cpu]; @@ -611,6 +605,9 @@ compute_cpu_load(int32 cpu) update_load_heaps(core); } + + if (gCPUEntries[cpu].fLoad > kVeryHighLoad) + sCurrentMode->rebalance_irqs(false); } @@ -1069,8 +1066,7 @@ static inline void update_cpu_performance(Thread* thread, int32 thisCore) { int32 load = max_c(thread->scheduler_data->load, - gCoreEntries[thisCore].fLoad); - load /= gCoreEntries[thisCore].fCPUCount; + get_core_load(&gCoreEntries[thisCore])); load = min_c(max_c(load, 0), kMaxLoad); if (load < kTargetLoad) { @@ -1369,6 +1365,8 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) gCPU[cpu].disabled = !enabled; + sCurrentMode->set_cpu_enabled(cpu, enabled); + CoreEntry* core = &gCoreEntries[gCPUToCore[cpu]]; PackageEntry* package = &gPackageEntries[gCPUToPackage[cpu]]; diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 3b13a9c371..b413a6bdba 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -16,6 +16,7 @@ struct scheduler_mode_operations { bool avoid_boost; void (*switch_to_mode)(void); + void (*set_cpu_enabled)(int32 cpu, bool enabled); bool (*has_cache_expired)(Thread* thread); int32 (*choose_core)(Thread* thread); bool (*should_rebalance)(Thread* thread); From 1e8ed5558dd69d1fbbd1578a2c5b2bfa321a5e35 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 25 Nov 2013 00:08:13 +0100 Subject: [PATCH 145/273] cpufreq: Rank modules and choose the best one --- headers/os/drivers/cpufreq.h | 2 ++ .../power/cpufreq/intel_pstates/intel_pstates.cpp | 2 ++ src/system/kernel/cpu.cpp | 13 +++++++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/headers/os/drivers/cpufreq.h b/headers/os/drivers/cpufreq.h index 7a4301feeb..ad36f9998a 100644 --- a/headers/os/drivers/cpufreq.h +++ b/headers/os/drivers/cpufreq.h @@ -17,6 +17,8 @@ const int kCPUPerformanceScaleMax = 1000; typedef struct cpufreq_module_info { module_info info; + float rank; + status_t (*increase_performance)(int delta, bool allowBoost); status_t (*decrease_performance)(int delta); } cpufreq_module_info; diff --git a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp index 7af294bf9c..f7f7fe1f9a 100644 --- a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp +++ b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp @@ -222,6 +222,8 @@ static cpufreq_module_info sIntelPStates = { std_ops, }, + 1.0f, + increase_performance, decrease_performance, }; diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 5ab8d26294..3b7a5ad804 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -63,13 +63,22 @@ load_cpufreq_module() while (true) { char name[B_FILE_NAME_LENGTH]; size_t nameLength = sizeof(name); + cpufreq_module_info* current = NULL; if (read_next_module_name(cookie, name, &nameLength) != B_OK) break; - if (get_module(name, (module_info**)&sCPUPerformanceModule) == B_OK) { + if (get_module(name, (module_info**)¤t) == B_OK) { dprintf("found cpufreq module: %s\n", name); - break; + + if (sCPUPerformanceModule != NULL) { + if (sCPUPerformanceModule->rank < current->rank) { + put_module(sCPUPerformanceModule->info.name); + sCPUPerformanceModule = current; + } else + put_module(name); + } else + sCPUPerformanceModule = current; } } From 26970784cd0050d72a1bb96eccd027dfc2769ffb Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 25 Nov 2013 00:17:42 +0100 Subject: [PATCH 146/273] scheduler: Clean scheduler_common.h scheduler_common.h is now meant for types, variables and functions used by both core scheduler code and implementations of scheduler modes. Functions like switch_thread() and update_thread_times() do not belong there anymore. --- src/system/kernel/scheduler/scheduler.cpp | 96 ++++++++++++++++++- .../kernel/scheduler/scheduler_common.h | 86 ----------------- 2 files changed, 91 insertions(+), 91 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index e1d7c45154..648794c89c 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1089,8 +1089,94 @@ update_cpu_performance(Thread* thread, int32 thisCore) } +/*! Switches the currently running thread. + This is a service function for scheduler implementations. + + \param fromThread The currently running thread. + \param toThread The thread to switch to. Must be different from + \a fromThread. +*/ +static inline void +switch_thread(Thread* fromThread, Thread* toThread) +{ + // notify the user debugger code + if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_unscheduled(fromThread); + + // stop CPU time based user timers + acquire_spinlock(&fromThread->team->time_lock); + acquire_spinlock(&fromThread->time_lock); + if (fromThread->HasActiveCPUTimeUserTimers() + || fromThread->team->HasActiveCPUTimeUserTimers()) { + user_timer_stop_cpu_timers(fromThread, toThread); + } + release_spinlock(&fromThread->time_lock); + release_spinlock(&fromThread->team->time_lock); + + // update CPU and Thread structures and perform the context switch + cpu_ent* cpu = fromThread->cpu; + toThread->previous_cpu = toThread->cpu = cpu; + fromThread->cpu = NULL; + cpu->running_thread = toThread; + cpu->previous_thread = fromThread; + + arch_thread_set_current_thread(toThread); + arch_thread_context_switch(fromThread, toThread); + + release_spinlock(&fromThread->cpu->previous_thread->scheduler_lock); + + // The use of fromThread below looks weird, but is correct. fromThread had + // been unscheduled earlier, but is back now. For a thread scheduled the + // first time the same is done in thread.cpp:common_thread_entry(). + + // continue CPU time based user timers + acquire_spinlock(&fromThread->team->time_lock); + acquire_spinlock(&fromThread->time_lock); + if (fromThread->HasActiveCPUTimeUserTimers() + || fromThread->team->HasActiveCPUTimeUserTimers()) { + user_timer_continue_cpu_timers(fromThread, cpu->previous_thread); + } + release_spinlock(&fromThread->time_lock); + release_spinlock(&fromThread->team->time_lock); + + // notify the user debugger code + if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_scheduled(fromThread); +} + + +static inline void +update_thread_times(Thread* oldThread, Thread* nextThread) +{ + bigtime_t now = system_time(); + if (oldThread == nextThread) { + acquire_spinlock(&oldThread->time_lock); + oldThread->kernel_time += now - oldThread->last_time; + oldThread->last_time = now; + release_spinlock(&oldThread->time_lock); + } else { + acquire_spinlock(&oldThread->time_lock); + oldThread->kernel_time += now - oldThread->last_time; + oldThread->last_time = 0; + release_spinlock(&oldThread->time_lock); + + acquire_spinlock(&nextThread->time_lock); + nextThread->last_time = now; + release_spinlock(&nextThread->time_lock); + } + + // If the old thread's team has user time timers, check them now. + Team* team = oldThread->team; + + acquire_spinlock(&team->time_lock); + if (team->HasActiveUserTimeUserTimers()) + user_timer_check_team_user_timers(team); + release_spinlock(&team->time_lock); +} + + static void -_scheduler_reschedule(void) +reschedule(void) { ASSERT(!are_interrupts_enabled()); @@ -1206,7 +1292,7 @@ _scheduler_reschedule(void) compute_thread_load(nextThread); // track kernel time (user time is tracked in thread_at_kernel_entry()) - scheduler_update_thread_times(oldThread, nextThread); + update_thread_times(oldThread, nextThread); // track CPU activity track_cpu_activity(oldThread, nextThread, thisCore); @@ -1235,7 +1321,7 @@ _scheduler_reschedule(void) modeLocker.Unlock(); if (nextThread != oldThread) - scheduler_switch_thread(oldThread, nextThread); + switch_thread(oldThread, nextThread); } } @@ -1253,7 +1339,7 @@ scheduler_reschedule(void) return; } - _scheduler_reschedule(); + reschedule(); } @@ -1299,7 +1385,7 @@ scheduler_start(void) { InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); - _scheduler_reschedule(); + reschedule(); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 190e968978..2f24441855 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -184,90 +184,4 @@ get_core_load(struct Scheduler::CoreEntry* core) } -/*! Switches the currently running thread. - This is a service function for scheduler implementations. - - \param fromThread The currently running thread. - \param toThread The thread to switch to. Must be different from - \a fromThread. -*/ -static inline void -scheduler_switch_thread(Thread* fromThread, Thread* toThread) -{ - // notify the user debugger code - if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) - user_debug_thread_unscheduled(fromThread); - - // stop CPU time based user timers - acquire_spinlock(&fromThread->team->time_lock); - acquire_spinlock(&fromThread->time_lock); - if (fromThread->HasActiveCPUTimeUserTimers() - || fromThread->team->HasActiveCPUTimeUserTimers()) { - user_timer_stop_cpu_timers(fromThread, toThread); - } - release_spinlock(&fromThread->time_lock); - release_spinlock(&fromThread->team->time_lock); - - // update CPU and Thread structures and perform the context switch - cpu_ent* cpu = fromThread->cpu; - toThread->previous_cpu = toThread->cpu = cpu; - fromThread->cpu = NULL; - cpu->running_thread = toThread; - cpu->previous_thread = fromThread; - - arch_thread_set_current_thread(toThread); - arch_thread_context_switch(fromThread, toThread); - - release_spinlock(&fromThread->cpu->previous_thread->scheduler_lock); - - // The use of fromThread below looks weird, but is correct. fromThread had - // been unscheduled earlier, but is back now. For a thread scheduled the - // first time the same is done in thread.cpp:common_thread_entry(). - - // continue CPU time based user timers - acquire_spinlock(&fromThread->team->time_lock); - acquire_spinlock(&fromThread->time_lock); - if (fromThread->HasActiveCPUTimeUserTimers() - || fromThread->team->HasActiveCPUTimeUserTimers()) { - user_timer_continue_cpu_timers(fromThread, cpu->previous_thread); - } - release_spinlock(&fromThread->time_lock); - release_spinlock(&fromThread->team->time_lock); - - // notify the user debugger code - if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) - user_debug_thread_scheduled(fromThread); -} - - -static inline void -scheduler_update_thread_times(Thread* oldThread, Thread* nextThread) -{ - bigtime_t now = system_time(); - if (oldThread == nextThread) { - acquire_spinlock(&oldThread->time_lock); - oldThread->kernel_time += now - oldThread->last_time; - oldThread->last_time = now; - release_spinlock(&oldThread->time_lock); - } else { - acquire_spinlock(&oldThread->time_lock); - oldThread->kernel_time += now - oldThread->last_time; - oldThread->last_time = 0; - release_spinlock(&oldThread->time_lock); - - acquire_spinlock(&nextThread->time_lock); - nextThread->last_time = now; - release_spinlock(&nextThread->time_lock); - } - - // If the old thread's team has user time timers, check them now. - Team* team = oldThread->team; - - acquire_spinlock(&team->time_lock); - if (team->HasActiveUserTimeUserTimers()) - user_timer_check_team_user_timers(team); - release_spinlock(&team->time_lock); -} - - #endif // KERNEL_SCHEDULER_COMMON_H From 0e94a12f8e0e5fe5ff5b2e3f83384f3586396c92 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 25 Nov 2013 00:35:15 +0100 Subject: [PATCH 147/273] kernel: Make CACHE_LINE_ALIGN visible in the whole kernel --- headers/private/kernel/arch/cpu.h | 2 ++ headers/private/kernel/arch/x86/arch_cpu.h | 2 ++ headers/private/kernel/cpu.h | 2 +- src/system/kernel/scheduler/scheduler_common.h | 3 --- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index e1bf1fdb5e..4d5754ef10 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -55,4 +55,6 @@ void arch_cpu_memory_read_write_barrier(void); #include +#define CACHE_LINE_ALIGN __attribute__((aligned(CACHE_LINE_SIZE))) + #endif /* _KERNEL_ARCH_CPU_H */ diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 33cf17e4ae..eb2e24a8cf 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -26,6 +26,8 @@ #define CPU_MAX_CACHE_LEVEL 8 +#define CACHE_LINE_SIZE 64 + // MSR registers (possibly Intel specific) #define IA32_MSR_TSC 0x10 diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 4a70428350..27fb6ff857 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -86,7 +86,7 @@ typedef struct cpu_ent { // arch-specific stuff arch_cpu_info arch; -} cpu_ent __attribute__((aligned(64))); +} cpu_ent CACHE_LINE_ALIGN; //extern cpu_ent gCPU[MAX_BOOT_CPUS]; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 2f24441855..dc07cf5bac 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -18,9 +18,6 @@ #include "RunQueue.h" -#define CACHE_LINE_ALIGN __attribute__((aligned(64))) - - //#define TRACE_SCHEDULER #ifdef TRACE_SCHEDULER # define TRACE(...) dprintf_no_syslog(__VA_ARGS__) From 7db89e8dc395db73368479fd9817b2b67899f3f6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 25 Nov 2013 23:50:27 +0100 Subject: [PATCH 148/273] kernel: Rework cpuidle module * Create new interface for cpuidle modules (similar to the cpufreq interface) * Generic cpuidle module is no longer needed * Fix and update Intel C-State module --- build/jam/packages/Haiku | 1 + build/jam/packages/HaikuBootstrap | 1 + headers/os/drivers/cpuidle.h | 46 +---- headers/private/kernel/arch/cpu.h | 1 - headers/private/kernel/arch/x86/arch_cpu.h | 29 ++- headers/private/kernel/cpu.h | 17 +- .../power/x86_cpuidle/intel_cpuidle.cpp | 121 ----------- src/add-ons/kernel/generic/Jamfile | 1 - src/add-ons/kernel/generic/cpuidle/Jamfile | 7 - .../kernel/generic/cpuidle/cpuidle.cpp | 105 ---------- src/add-ons/kernel/power/Jamfile | 1 + src/add-ons/kernel/power/cpuidle/Jamfile | 4 + .../power/cpuidle/intel_cstates/Jamfile | 8 + .../cpuidle/intel_cstates/intel_cstates.cpp | 195 ++++++++++++++++++ src/system/kernel/UserTimer.cpp | 2 +- src/system/kernel/arch/x86/arch_cpu.cpp | 37 +--- .../kernel/arch/x86/arch_debug_console.cpp | 4 +- src/system/kernel/cpu.cpp | 65 +++++- src/system/kernel/debug/debug.cpp | 4 +- src/system/kernel/main.cpp | 2 +- src/system/kernel/smp.cpp | 34 +-- src/system/kernel/timer.cpp | 10 +- 22 files changed, 356 insertions(+), 339 deletions(-) delete mode 100644 src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp delete mode 100644 src/add-ons/kernel/generic/cpuidle/Jamfile delete mode 100644 src/add-ons/kernel/generic/cpuidle/cpuidle.cpp create mode 100644 src/add-ons/kernel/power/cpuidle/Jamfile create mode 100644 src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile create mode 100644 src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp diff --git a/build/jam/packages/Haiku b/build/jam/packages/Haiku index 46df98aee6..5511bc4aaa 100644 --- a/build/jam/packages/Haiku +++ b/build/jam/packages/Haiku @@ -53,6 +53,7 @@ AddFilesToPackage add-ons kernel partitioning_systems : amiga_rdb apple efi_gpt intel session ; AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ; AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ; +AddFilesToPackage add-ons kernel power cpuidle : intel_cstates@x86,x86_64 ; if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 { AddFilesToPackage add-ons kernel cpu : generic_x86 ; diff --git a/build/jam/packages/HaikuBootstrap b/build/jam/packages/HaikuBootstrap index 1fafbb20fe..24d6ce63f2 100644 --- a/build/jam/packages/HaikuBootstrap +++ b/build/jam/packages/HaikuBootstrap @@ -52,6 +52,7 @@ AddFilesToPackage add-ons kernel partitioning_systems : amiga_rdb apple efi_gpt intel session ; AddFilesToPackage add-ons kernel interrupt_controllers : openpic@ppc ; AddFilesToPackage add-ons kernel power cpufreq : intel_pstates@x86,x86_64 ; +AddFilesToPackage add-ons kernel power cpuidle : intel_cstates@x86,x86_64 ; if $(TARGET_ARCH) = x86 || $(TARGET_ARCH) = x86_64 { AddFilesToPackage add-ons kernel cpu : generic_x86 ; diff --git a/headers/os/drivers/cpuidle.h b/headers/os/drivers/cpuidle.h index 7649cdadab..c22b3e6cc6 100644 --- a/headers/os/drivers/cpuidle.h +++ b/headers/os/drivers/cpuidle.h @@ -8,50 +8,18 @@ #include -#ifdef __cplusplus -extern "C" { -#endif -#define CPUIDLE_CSTATE_MAX 8 -#define CSTATE_NAME_LENGTH 32 -#define B_CPUIDLE_MODULE_NAME "generic/cpuidle/v1" +#define CPUIDLE_MODULES_PREFIX "power/cpuidle" -struct CpuidleStat { - uint64 usageCount; - bigtime_t usageTime; -}; +typedef struct cpuidle_module_info { + module_info info; + float rank; -struct CpuidleInfo { - int32 cstateSleep; - CpuidleStat stats[CPUIDLE_CSTATE_MAX]; -}; + void (*idle)(void); + void (*wait)(int32* variable, int32 test); +} cpuidle_module_info; -struct CpuidleDevice; - -struct CpuidleCstate { - char name[CSTATE_NAME_LENGTH]; - int32 latency; - int32 (*EnterIdle)(int32 state, CpuidleDevice *device); - void *pData; -}; - - -struct CpuidleDevice { - CpuidleCstate cStates[CPUIDLE_CSTATE_MAX]; - int32 cStateCount; -}; - - -struct CpuidleModuleInfo { - module_info info; - status_t (*AddDevice)(CpuidleDevice *device); -}; - - -#ifdef __cplusplus -} -#endif #endif // _CPUIDLE_MODULE_H diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index 4d5754ef10..d5ea2de81e 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -41,7 +41,6 @@ ssize_t arch_cpu_user_strlcpy(char *to, const char *from, size_t size, status_t arch_cpu_user_memset(void *s, char c, size_t count, addr_t *faultHandler); -void arch_cpu_idle(void); void arch_cpu_sync_icache(void *address, size_t length); void arch_cpu_memory_read_barrier(void); diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index eb2e24a8cf..f4ccaf487c 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -185,6 +185,10 @@ | IA32_FEATURE_AMD_EXT_RDTSCP \ | IA32_FEATURE_AMD_EXT_LONG) +// x86 defined features from cpuid eax 5, ecx register +#define IA32_FEATURE_POWER_MWAIT (1 << 0) +#define IA32_FEATURE_INTERRUPT_MWAIT (1 << 1) + // x86 defined features from cpuid eax 6, eax register // reference http://www.intel.com/Assets/en_US/PDF/appnote/241618.pdf (Table 5-11) #define IA32_FEATURE_DTS (1 << 0) //Digital Thermal Sensor @@ -199,6 +203,9 @@ #define IA32_FEATURE_APERFMPERF (1 << 0) //IA32_APERF, IA32_MPERF #define IA32_FEATURE_EPB (1 << 3) //IA32_ENERGY_PERF_BIAS +// x86 defined features from cpuid eax 0x80000007, edx register +#define IA32_FEATURE_INVARIANT_TSC (1 << 8) + // cr4 flags #define IA32_CR4_PAE (1UL << 5) #define IA32_CR4_GLOBAL_PAGES (1UL << 7) @@ -282,8 +289,10 @@ enum x86_feature_type { FEATURE_EXT, // cpuid eax=1, edx register FEATURE_EXT_AMD_ECX, // cpuid eax=0x80000001, ecx register (AMD) FEATURE_EXT_AMD, // cpuid eax=0x80000001, edx register (AMD) + FEATURE_5_ECX, // cpuid eax=5, ecx register FEATURE_6_EAX, // cpuid eax=6, eax registers FEATURE_6_ECX, // cpuid eax=6, ecx registers + FEATURE_EXT_7_EDX, // cpuid eax=0x80000007, edx register FEATURE_NUM }; @@ -330,9 +339,6 @@ typedef struct arch_cpu_info { } arch_cpu_info; -#undef PAUSE -#define PAUSE() asm volatile ("pause;") - #define nop() __asm__ ("nop"::) #define x86_read_cr0() ({ \ @@ -426,6 +432,9 @@ typedef struct arch_cpu_info { }) +extern void (*gCpuIdleFunc)(void); + + #ifdef __cplusplus extern "C" { #endif @@ -478,6 +487,20 @@ void x86_fnsave_swap(void* oldFpuState, const void* newFpuState); #endif +static inline void +arch_cpu_idle(void) +{ + gCpuIdleFunc(); +} + + +static inline void +arch_cpu_pause(void) +{ + asm volatile("pause"); +} + + #ifdef __cplusplus } // extern "C" { #endif diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 27fb6ff857..3337d387d4 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -17,12 +17,6 @@ #include -// define PAUSE, if not done in arch/cpu.h -#ifndef PAUSE -# define PAUSE() -#endif - - struct kernel_args; namespace BKernel { @@ -114,6 +108,17 @@ cpu_topology_node* get_cpu_topology(void); status_t increase_cpu_performance(int delta, bool allowBoost); status_t decrease_cpu_performance(int delta); +void cpu_idle(void); +void cpu_wait(int32* variable, int32 test); + + +static inline void +cpu_pause(void) +{ + arch_cpu_pause(); +} + + void _user_clear_caches(void *address, size_t length, uint32 flags); bool _user_cpu_enabled(int32 cpu); status_t _user_set_cpu_enabled(int32 cpu, bool enabled); diff --git a/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp b/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp deleted file mode 100644 index cbbd1bc988..0000000000 --- a/src/add-ons/kernel/drivers/power/x86_cpuidle/intel_cpuidle.cpp +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2012, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Yongcong Du - */ - -#include -#include -#include - -#include - -#include - -#include "x86_cpuidle.h" - -static CpuidleDevice sIntelDevice; - -static void *kMwaitEax[] = { - // C0, we never use it - (void *)0x00, - // MWAIT C1 - (void *)0x00, - // MWAIT C2 - (void *)0x10, - // MWAIT C3 - (void *)0x20, - // MWAIT C4 - (void *)0x30, - // MWAIT C5 - (void *)0x40, - // MWAIT C6, 0x2 is used to fully shrink L2 cache - (void *)0x52 -}; - - -static int32 -IntelCstateIdleEnter(int32 state, CpuidleDevice *device) -{ - cpu_ent *cpu = get_cpu_struct(); - if (cpu->invoke_scheduler) - return 0; - - CpuidleCstate *cState = &device->cStates[state]; - x86_monitor((void *)&cpu->invoke_scheduler, 0, 0); - if (!cpu->invoke_scheduler) - x86_mwait((unsigned long)cState->pData, 1); - - return state; -} - - -static CpuidleCstate sSnbcStates[CPUIDLE_CSTATE_MAX] = { - {}, - { - "C1-SNB", - 1, - IntelCstateIdleEnter, - }, - { - "C3-SNB", - 80, - IntelCstateIdleEnter, - }, - { - "C6-SNB", - 104, - IntelCstateIdleEnter, - }, - { - "C7-SNB", - 109, - IntelCstateIdleEnter, - }, -}; - - -status_t -intel_cpuidle_init(void) -{ - dprintf("intel idle init\n"); - cpu_ent *cpu = get_cpu_struct(); - if (cpu->arch.vendor != VENDOR_INTEL || cpu->arch.family != 6) - return B_ERROR; - - // Calculated Model Value: M = (Extended Model << 4) + Model - uint32 model = (cpu->arch.extended_model << 4) + cpu->arch.model; - if (model != 0x2a && model != 0x2d) - return B_ERROR; - - cpuid_info cpuid; - get_current_cpuid(&cpuid, 5, 0); - /* ecx[0] monitor/mwait extension supported - * ecx[1] support for treating interrupts as break-events for mwait - * edx number of sub-states - */ - if ((cpuid.regs.ecx & 0x1) == 0 || - (cpuid.regs.ecx & 0x2) == 0 || - cpuid.regs.edx == 0) { - return B_ERROR; - } - - sIntelDevice.cStateCount = 1; - for (int32 i = 1; i < CPUIDLE_CSTATE_MAX; i++) { - int32 subStates = (cpuid.regs.edx >> ((i) * 4)) & 0xf; - // no sub-states means the state is not available - if (!subStates) - continue; - sIntelDevice.cStates[sIntelDevice.cStateCount] = - sSnbcStates[i]; - sIntelDevice.cStates[sIntelDevice.cStateCount].pData = - kMwaitEax[i]; - sIntelDevice.cStateCount++; - } - status_t status = gIdle->AddDevice(&sIntelDevice); - if (status == B_OK) - dprintf("using intel idle\n"); - return status; -} diff --git a/src/add-ons/kernel/generic/Jamfile b/src/add-ons/kernel/generic/Jamfile index 9a1677d11b..1ac20bafc0 100644 --- a/src/add-ons/kernel/generic/Jamfile +++ b/src/add-ons/kernel/generic/Jamfile @@ -3,7 +3,6 @@ SubDir HAIKU_TOP src add-ons kernel generic ; SubInclude HAIKU_TOP src add-ons kernel generic ata_adapter ; SubInclude HAIKU_TOP src add-ons kernel generic atomizer ; SubInclude HAIKU_TOP src add-ons kernel generic bios ; -SubInclude HAIKU_TOP src add-ons kernel generic cpuidle ; SubInclude HAIKU_TOP src add-ons kernel generic dpc ; SubInclude HAIKU_TOP src add-ons kernel generic ide_adapter ; SubInclude HAIKU_TOP src add-ons kernel generic locked_pool ; diff --git a/src/add-ons/kernel/generic/cpuidle/Jamfile b/src/add-ons/kernel/generic/cpuidle/Jamfile deleted file mode 100644 index bb7cdd2746..0000000000 --- a/src/add-ons/kernel/generic/cpuidle/Jamfile +++ /dev/null @@ -1,7 +0,0 @@ -SubDir HAIKU_TOP src add-ons kernel generic cpuidle ; - -UsePrivateKernelHeaders ; - -KernelAddon cpuidle : - cpuidle.cpp - ; diff --git a/src/add-ons/kernel/generic/cpuidle/cpuidle.cpp b/src/add-ons/kernel/generic/cpuidle/cpuidle.cpp deleted file mode 100644 index cc17be8fac..0000000000 --- a/src/add-ons/kernel/generic/cpuidle/cpuidle.cpp +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2012, Haiku, Inc. All Rights Reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Yongcong Du - */ - -#include -#include -#include - -#include -#include - - -static CpuidleInfo sPerCPU[B_MAX_CPU_COUNT]; -static CpuidleDevice *sDevice; -extern void (*gCpuIdleFunc)(void); - - -/* - * next cstate selection algorithm is based on NetBSD's - * it's simple, stupid - */ -static int32 -SelectCstate(CpuidleInfo *info) -{ - static const int32 csFactor = 3; - - for (int32 i = sDevice->cStateCount - 1; i > 0; i--) { - CpuidleCstate *cState = &sDevice->cStates[i]; - if (info->cstateSleep > cState->latency * csFactor) - return i; - } - - /* Choose C1 if there's no state found */ - return 1; -} - - -static inline void -EnterCstate(int32 state, CpuidleInfo *info) -{ - CpuidleCstate *cstate = &sDevice->cStates[state]; - bigtime_t now = system_time(); - int32 finalState = cstate->EnterIdle(state, sDevice); - if (finalState > 0) { - bigtime_t diff = system_time() - now; - info->cstateSleep = diff; - info->stats[finalState].usageCount++; - info->stats[finalState].usageTime += diff; - } else { - info->cstateSleep = 0; - } -} - - -static void -CpuCstateIdle(void) -{ - CpuidleInfo *info = &sPerCPU[smp_get_current_cpu()]; - int32 state = SelectCstate(info); - EnterCstate(state, info); -} - - -static status_t -std_ops(int32 op, ...) -{ - switch (op) { - case B_MODULE_INIT: - return B_OK; - case B_MODULE_UNINIT: - return B_OK; - } - - return B_ERROR; -} - - -status_t AddDevice(CpuidleDevice *device) -{ - sDevice = device; - memory_write_barrier(); - gCpuIdleFunc = CpuCstateIdle; - return B_OK; -} - - -static CpuidleModuleInfo sCpuidleModule = { - { - B_CPUIDLE_MODULE_NAME, - 0, - std_ops - }, - - AddDevice, -}; - - -module_info *modules[] = { - (module_info *)&sCpuidleModule, - NULL -}; diff --git a/src/add-ons/kernel/power/Jamfile b/src/add-ons/kernel/power/Jamfile index 84032da183..289cb1f2bc 100644 --- a/src/add-ons/kernel/power/Jamfile +++ b/src/add-ons/kernel/power/Jamfile @@ -1,4 +1,5 @@ SubDir HAIKU_TOP src add-ons kernel power ; SubInclude HAIKU_TOP src add-ons kernel power cpufreq ; +SubInclude HAIKU_TOP src add-ons kernel power cpuidle ; diff --git a/src/add-ons/kernel/power/cpuidle/Jamfile b/src/add-ons/kernel/power/cpuidle/Jamfile new file mode 100644 index 0000000000..702f538988 --- /dev/null +++ b/src/add-ons/kernel/power/cpuidle/Jamfile @@ -0,0 +1,4 @@ +SubDir HAIKU_TOP src add-ons kernel power cpuidle ; + +SubInclude HAIKU_TOP src add-ons kernel power cpuidle intel_cstates ; + diff --git a/src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile b/src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile new file mode 100644 index 0000000000..4c8ee487d7 --- /dev/null +++ b/src/add-ons/kernel/power/cpuidle/intel_cstates/Jamfile @@ -0,0 +1,8 @@ +SubDir HAIKU_TOP src add-ons kernel power cpuidle intel_cstates ; + +UsePrivateKernelHeaders ; + +KernelAddon intel_cstates : + intel_cstates.cpp + ; + diff --git a/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp new file mode 100644 index 0000000000..d401b451a6 --- /dev/null +++ b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp @@ -0,0 +1,195 @@ +/* + * Copyright 2012-2013, Haiku, Inc. All Rights Reserved. + * Distributed under the terms of the MIT License. + * + * Authors: + * Yongcong Du + * Paweł Dziepak, + */ + + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + + +#define CPUIDLE_CSTATE_MAX 8 + +#define MWAIT_INTERRUPTS_BREAK (1 << 0) + +#define INTEL_CSTATES_MODULE_NAME CPUIDLE_MODULES_PREFIX "/intel_cstates/v1" + +#define TIME_STEP 250 + +struct CState { + uint32 fCode; + int fSubStatesCount; + int fSubStatesStep; +}; + +static CState sCStates[CPUIDLE_CSTATE_MAX]; +static int sCStateCount; + +static bigtime_t* sIdleTime; + + +static inline void +x86_monitor(void* address, uint32 ecx, uint32 edx) +{ + asm volatile("monitor" : : "a" (address), "c" (ecx), "d"(edx)); +} + + +static inline void +x86_mwait(uint32 eax, uint32 ecx) +{ + asm volatile("mwait" : : "a" (eax), "c" (ecx)); +} + + +static void +idle(void) +{ + ASSERT(thread_get_current_thread()->pinned_to_cpu > 0); + int32 cpu = smp_get_current_cpu(); + + bigtime_t idleTime = sIdleTime[cpu]; + int state = min_c(idleTime / TIME_STEP, sCStateCount - 1); + + ASSERT(state >= 0 && state < sCStateCount); + + int subState = idleTime % TIME_STEP; + subState /= sCStates[state].fSubStatesStep; + + ASSERT(subState >= 0 && subState < sCStates[state].fSubStatesCount); + + InterruptsLocker locker; + int dummy; + bigtime_t start = system_time(); + x86_monitor(&dummy, 0, 0); + x86_mwait(sCStates[state].fCode | subState, MWAIT_INTERRUPTS_BREAK); + bigtime_t delta = system_time() - start; + locker.Unlock(); + + sIdleTime[cpu] = (idleTime + delta) / 2; +} + + +static void +wait(int32* variable, int32 test) +{ + InterruptsLocker _; + x86_monitor(variable, 0, 0); + if (*variable != test) + x86_mwait(sCStates[0].fCode, MWAIT_INTERRUPTS_BREAK); +} + + +static status_t +init_cstates() +{ + if (!x86_check_feature(IA32_FEATURE_EXT_MONITOR, FEATURE_EXT)) + return B_ERROR; + if (!x86_check_feature(IA32_FEATURE_POWER_MWAIT, FEATURE_5_ECX)) + return B_ERROR; + if (!x86_check_feature(IA32_FEATURE_INTERRUPT_MWAIT, FEATURE_5_ECX)) + return B_ERROR; + + // we need invariant TSC + if (!x86_check_feature(IA32_FEATURE_INVARIANT_TSC, FEATURE_EXT_7_EDX)) + return B_ERROR; + + // get C-state data + cpuid_info cpuid; + get_current_cpuid(&cpuid, 0, 0); + uint32 maxBasicLeaf = cpuid.eax_0.max_eax; + if (maxBasicLeaf < 5) + return B_ERROR; + + get_current_cpuid(&cpuid, 5, 0); + if ((cpuid.regs.eax & 0xffff) < sizeof(int32)) + return B_ERROR; + + char cStates[64]; + unsigned int offset = 0; + for (int32 i = 1; i < CPUIDLE_CSTATE_MAX; i++) { + int32 subStates = (cpuid.regs.edx >> (i * 4)) & 0xf; + // no sub-states means the state is not available + if (subStates == 0) + continue; + + if (offset < sizeof(cStates)) { + offset += snprintf(cStates + offset, sizeof(cStates) - offset, + ", C%" B_PRId32, i); + } + + sCStates[sCStateCount].fCode = sCStateCount * 0x10; + sCStates[sCStateCount].fSubStatesCount = subStates; + sCStates[sCStateCount].fSubStatesStep = TIME_STEP / subStates; + sCStateCount++; + } + + if (sCStateCount == 0) + return B_ERROR; + + sIdleTime = new(std::nothrow) bigtime_t[smp_get_num_cpus()]; + if (sIdleTime == NULL) + return B_NO_MEMORY; + memset(sIdleTime, 0, sizeof(bigtime_t) * smp_get_num_cpus()); + + dprintf("using Intel C-States: C0%s\n", cStates); + return B_OK; +} + + +static status_t +uninit_cstates() +{ + delete[] sIdleTime; + return B_OK; +} + + +static status_t +std_ops(int32 op, ...) +{ + switch (op) { + case B_MODULE_INIT: + return init_cstates(); + + case B_MODULE_UNINIT: + uninit_cstates(); + return B_OK; + } + + return B_ERROR; +} + + +static cpuidle_module_info sIntelCStates = { + { + INTEL_CSTATES_MODULE_NAME, + 0, + std_ops, + }, + + 0.8f, + + idle, + wait +}; + + +module_info* modules[] = { + (module_info*)&sIntelCStates, + NULL +}; + diff --git a/src/system/kernel/UserTimer.cpp b/src/system/kernel/UserTimer.cpp index b3da871849..db816a426f 100644 --- a/src/system/kernel/UserTimer.cpp +++ b/src/system/kernel/UserTimer.cpp @@ -196,7 +196,7 @@ UserTimer::HandleTimerHook(struct timer* timer) while (!locked && atomic_get(&userTimer->fSkip) == 0) { locked = try_acquire_write_seqlock(&sUserTimerLock); if (!locked) - PAUSE(); + cpu_pause(); } if (locked) { diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index be1afdae0a..16d60bfe96 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -1,5 +1,6 @@ /* * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2012, Alex Smith, alex@alex-smith.me.uk. * Distributed under the terms of the MIT License. * @@ -69,16 +70,6 @@ static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = { #define K8_CMPHALT (K8_SMIONCMPHALT | K8_C1EONCMPHALT) -/* - * 0 favors highest performance while 15 corresponds to the maximum energy - * savings. 7 means balance between performance and energy savings. - * Refer to Section 14.3.4 in for details - */ -#define ENERGY_PERF_BIAS_PERFORMANCE 0 -#define ENERGY_PERF_BIAS_BALANCE 7 -#define ENERGY_PERF_BIAS_POWERSAVE 15 - struct set_mtrr_parameter { int32 index; uint64 base; @@ -929,12 +920,22 @@ detect_cpu(int currentCPU) cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT; } + if (maxBasicLeaf >= 5) { + get_current_cpuid(&cpuid, 5, 0); + cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx; + } + if (maxBasicLeaf >= 6) { get_current_cpuid(&cpuid, 6, 0); cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax; cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx; } + if (maxExtendedLeaf >= 0x80000007) { + get_current_cpuid(&cpuid, 0x80000007, 0); + cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx; + } + detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf); #if DUMP_FEATURE_STRING @@ -1060,15 +1061,6 @@ arch_cpu_init_percpu(kernel_args* args, int cpu) gCpuIdleFunc = halt_idle; } - if (x86_check_feature(IA32_FEATURE_EPB, FEATURE_6_ECX)) { - uint64 msr = x86_read_msr(IA32_MSR_ENERGY_PERF_BIAS); - if ((msr & 0xf) == ENERGY_PERF_BIAS_PERFORMANCE) { - msr &= ~0xf; - msr |= ENERGY_PERF_BIAS_BALANCE; - x86_write_msr(IA32_MSR_ENERGY_PERF_BIAS, msr); - } - } - return B_OK; } @@ -1287,13 +1279,6 @@ arch_cpu_shutdown(bool rebootSystem) } -void -arch_cpu_idle(void) -{ - gCpuIdleFunc(); -} - - void arch_cpu_sync_icache(void* address, size_t length) { diff --git a/src/system/kernel/arch/x86/arch_debug_console.cpp b/src/system/kernel/arch/x86/arch_debug_console.cpp index 8309a9967b..1f226dbc56 100644 --- a/src/system/kernel/arch/x86/arch_debug_console.cpp +++ b/src/system/kernel/arch/x86/arch_debug_console.cpp @@ -316,7 +316,7 @@ arch_debug_blue_screen_getchar(void) if (c >= 0) return (char)c; - PAUSE(); + arch_cpu_pause(); } } @@ -352,7 +352,7 @@ arch_debug_serial_getchar(void) if ((lineStatus & 0x1) != 0) break; - PAUSE(); + arch_cpu_pause(); } return in8(sSerialBasePort + SERIAL_RECEIVE_BUFFER); diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 3b7a5ad804..2fdf6fd1da 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -30,6 +31,7 @@ uint32 gCPUCacheLevelCount; static cpu_topology_node sCPUTopology; static cpufreq_module_info* sCPUPerformanceModule; +static cpuidle_module_info* sCPUIdleModule; static spinlock sSetCpuLock; @@ -89,6 +91,40 @@ load_cpufreq_module() } +static void +load_cpuidle_module() +{ + void* cookie = open_module_list(CPUIDLE_MODULES_PREFIX); + + while (true) { + char name[B_FILE_NAME_LENGTH]; + size_t nameLength = sizeof(name); + cpuidle_module_info* current = NULL; + + if (read_next_module_name(cookie, name, &nameLength) != B_OK) + break; + + if (get_module(name, (module_info**)¤t) == B_OK) { + dprintf("found cpuidle module: %s\n", name); + + if (sCPUIdleModule != NULL) { + if (sCPUIdleModule->rank < current->rank) { + put_module(sCPUIdleModule->info.name); + sCPUIdleModule = current; + } else + put_module(name); + } else + sCPUIdleModule = current; + } + } + + close_module_list(cookie); + + if (sCPUIdleModule == NULL) + dprintf("no valid cpuidle module found\n"); +} + + status_t cpu_init_post_modules(kernel_args *args) { @@ -97,6 +133,7 @@ cpu_init_post_modules(kernel_args *args) return result; load_cpufreq_module(); + load_cpuidle_module(); return B_OK; } @@ -232,6 +269,8 @@ cpu_build_topology_tree(void) int32 lastID[CPU_TOPOLOGY_LEVELS]; memset(&lastID, 0, sizeof(lastID)); cpu_rebuild_topology_tree(&sCPUTopology, lastID); + + return B_OK; } @@ -260,6 +299,31 @@ decrease_cpu_performance(int delta) } +void +cpu_idle(void) +{ +#if KDEBUG + if (!are_interrupts_enabled()) + panic("cpu_idle() called with interrupts disabled."); +#endif + + if (sCPUIdleModule != NULL) + sCPUIdleModule->idle(); + else + arch_cpu_idle(); +} + + +void +cpu_wait(int32* variable, int32 test) +{ + if (sCPUIdleModule != NULL) + sCPUIdleModule->wait(variable, test); + else + arch_cpu_pause(); +} + + // #pragma mark - @@ -283,7 +347,6 @@ _user_cpu_enabled(int32 cpu) status_t _user_set_cpu_enabled(int32 cpu, bool enabled) { - cpu_status state; int32 i, count; if (cpu < 0 || cpu >= smp_get_num_cpus()) diff --git a/src/system/kernel/debug/debug.cpp b/src/system/kernel/debug/debug.cpp index e75acd66dc..b56145b893 100644 --- a/src/system/kernel/debug/debug.cpp +++ b/src/system/kernel/debug/debug.cpp @@ -703,7 +703,7 @@ kgetc(void) } } - PAUSE(); + cpu_pause(); } } @@ -1008,7 +1008,7 @@ hand_over_kernel_debugger() // initiated by the user, it is harmless, though. sHandOverKDL = true; while (atomic_get(&sHandOverKDLToCPU) >= 0) - PAUSE(); + cpu_wait(&sHandOverKDLToCPU, -1); } diff --git a/src/system/kernel/main.cpp b/src/system/kernel/main.cpp index 9dedf47db3..f13526542b 100644 --- a/src/system/kernel/main.cpp +++ b/src/system/kernel/main.cpp @@ -262,7 +262,7 @@ _start(kernel_args *bootKernelArgs, int currentCPU) #endif for (;;) - arch_cpu_idle(); + cpu_idle(); return 0; } diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index a07df3f290..0477b85129 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -358,7 +358,7 @@ acquire_spinlock(spinlock* lock) } process_all_pending_ici(currentCPU); - PAUSE(); + cpu_wait(lock, 0); } if (atomic_get_and_set((int32*)lock, 1) == 0) break; @@ -411,7 +411,7 @@ acquire_spinlock_nocheck(spinlock *lock) count = 0; } - PAUSE(); + cpu_wait(lock, 0); } if (atomic_get_and_set((int32*)lock, 1) == 0) @@ -455,7 +455,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) } process_all_pending_ici(currentCPU); - PAUSE(); + cpu_wait(lock, 0); } if (atomic_get_and_set((int32*)lock, 1) == 0) break; @@ -570,7 +570,7 @@ acquire_write_spinlock(rw_spinlock* lock) } process_all_pending_ici(currentCPU); - PAUSE(); + cpu_wait(&lock->lock, 0); } } } @@ -583,7 +583,7 @@ release_write_spinlock(rw_spinlock* lock) uint32 previous = atomic_get_and_set(&lock->lock, 0); if ((previous & 1u << 31) == 0) { panic("release_write_spinlock: lock %p was already released (value: " - "%x)\n", lock, previous); + "%#" B_PRIx32 ")\n", lock, previous); } #else atomic_set(&lock->lock, 0); @@ -639,7 +639,7 @@ acquire_read_spinlock(rw_spinlock* lock) } process_all_pending_ici(currentCPU); - PAUSE(); + cpu_wait(&lock->lock, 0); } } } @@ -652,7 +652,7 @@ release_read_spinlock(rw_spinlock* lock) uint32 previous = atomic_add(&lock->lock, -1); if ((previous & 1 << 31) != 0) { panic("release_read_spinlock: lock %p was already released (value:" - " %x)\n", lock, previous); + " %#" B_PRIx32 ")\n", lock, previous); } #else atomic_add(&lock->lock, -1); @@ -695,7 +695,7 @@ release_read_seqlock(seqlock* lock, uint32 count) { uint32 current = atomic_get((int32*)&lock->count); if (count % 2 == 1 || current != count) { - PAUSE(); + cpu_pause(); return false; } @@ -719,7 +719,7 @@ retry: state = disable_interrupts(); process_all_pending_ici(smp_get_current_cpu()); restore_interrupts(state); - PAUSE(); + cpu_pause(); } state = disable_interrupts(); acquire_spinlock(&sFreeMessageSpinlock); @@ -757,7 +757,7 @@ find_free_message_interrupts_disabled(int32 currentCPU, while (sFreeMessageCount <= 0) { release_spinlock(&sFreeMessageSpinlock); process_all_pending_ici(currentCPU); - PAUSE(); + cpu_pause(); acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); } @@ -1014,7 +1014,7 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie) // wait for all CPUs to finish while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0) - PAUSE(); + cpu_pause(); } function(cookie, 0); @@ -1085,7 +1085,7 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, // if the message is sync after it has removed it from the mailbox while (msg->done == false) { process_all_pending_ici(currentCPU); - PAUSE(); + cpu_pause(); } // for SYNC messages, it's our responsibility to put it // back into the free list @@ -1148,7 +1148,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, // if the message is sync after it has removed it from the mailbox while (msg->done == false) { process_all_pending_ici(currentCPU); - PAUSE(); + cpu_pause(); } // for SYNC messages, it's our responsibility to put it @@ -1210,7 +1210,7 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, while (msg->done == false) { process_all_pending_ici(currentCPU); - PAUSE(); + cpu_pause(); } TRACE(("smp_send_broadcast_ici: returning message to free list\n")); @@ -1274,7 +1274,7 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, while (msg->done == false) { process_all_pending_ici(currentCPU); - PAUSE(); + cpu_pause(); } TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: returning " @@ -1310,7 +1310,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous) if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0) process_early_cpu_call(cpu); - PAUSE(); + cpu_pause(); } return false; @@ -1346,7 +1346,7 @@ smp_cpu_rendezvous(uint32* var, int current_cpu) uint32 allReady = ((uint32)1 << sNumCPUs) - 1; while ((uint32)atomic_get((int32*)var) != allReady) - PAUSE(); + cpu_wait((int32*)var, allReady); } diff --git a/src/system/kernel/timer.cpp b/src/system/kernel/timer.cpp index d10e3c3fa0..5a3ea20969 100644 --- a/src/system/kernel/timer.cpp +++ b/src/system/kernel/timer.cpp @@ -447,9 +447,8 @@ cancel_timer(timer* event) if (cpu != smp_get_current_cpu()) { spinLocker.Unlock(); - while (atomic_get(&cpuData.current_event_in_progress) == 1) { - PAUSE(); - } + while (atomic_get(&cpuData.current_event_in_progress) == 1) + cpu_wait(&cpuData.current_event_in_progress, 0); } return true; @@ -461,7 +460,6 @@ spin(bigtime_t microseconds) { bigtime_t time = system_time(); - while ((system_time() - time) < microseconds) { - PAUSE(); - } + while ((system_time() - time) < microseconds) + cpu_pause(); } From 35153d1c49c6b0bb3858c24e8824cbc834973c07 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 26 Nov 2013 03:03:41 +0100 Subject: [PATCH 149/273] scheduler: Send reschedule ICI after enqueueing the thread --- src/system/kernel/scheduler/scheduler.cpp | 55 +++++++++++------------ 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 648794c89c..d421f81dbb 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -548,7 +548,7 @@ choose_cpu(int32 core) } -static bool +static void choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) { if (targetCore == -1 && targetCPU != -1) @@ -562,19 +562,6 @@ choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) ASSERT(targetCore >= 0 && targetCore < gRunQueueCount); ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); - - int32 targetPriority = gCPUEntries[targetCPU].fPriority; - int32 threadPriority = get_effective_priority(thread); - - if (threadPriority > targetPriority) { - // It is possible that another CPU schedules the thread before the - // target CPU. However, since the target CPU is sent an ICI it will - // reschedule anyway and update its heap key to the correct value. - update_cpu_priority(targetCPU, threadPriority); - return true; - } - - return false; } @@ -716,7 +703,7 @@ enqueue(Thread* thread, bool newOne) targetCore = schedulerThreadData->previous_core; } - bool shouldReschedule = choose_core_and_cpu(thread, targetCore, targetCPU); + choose_core_and_cpu(thread, targetCore, targetCPU); schedulerThreadData->previous_core = targetCore; TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", @@ -734,7 +721,14 @@ enqueue(Thread* thread, bool newOne) NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - if (shouldReschedule) { + int32 targetPriority = gCPUEntries[targetCPU].fPriority; + + if (threadPriority > targetPriority) { + // It is possible that another CPU schedules the thread before the + // target CPU. However, since the target CPU is sent an ICI it will + // reschedule anyway and update its heap key to the correct value. + update_cpu_priority(targetCPU, threadPriority); + if (targetCPU == smp_get_current_cpu()) gCPU[targetCPU].invoke_scheduler = true; else { @@ -1638,7 +1632,7 @@ create_debug_heaps() static status_t -_scheduler_init() +init() { // create logical processor to core and package mappings int32 cpuCount, coreCount, packageCount; @@ -1751,18 +1745,6 @@ _scheduler_init() if (result != B_OK) return result; - scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); - - add_debugger_command_etc("run_queue", &dump_run_queue, - "List threads in run queue", "\nLists threads in run queue", 0); - add_debugger_command_etc("cpu_heap", &dump_cpu_heap, - "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", - 0); - if (!gSingleCore) { - add_debugger_command_etc("idle_cores", &dump_idle_cores, - "List idle cores", "\nList idle cores", 0); - } - runQueuesDeleter.Detach(); pinnedRunQueuesDeleter.Detach(); coreHighLoadHeapDeleter.Detach(); @@ -1784,10 +1766,22 @@ scheduler_init(void) " cache level%s\n", cpuCount, cpuCount != 1 ? "s" : "", gCPUCacheLevelCount, gCPUCacheLevelCount != 1 ? "s" : ""); - status_t result = _scheduler_init(); + status_t result = init(); if (result != B_OK) panic("scheduler_init: failed to initialize scheduler\n"); + scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); + + add_debugger_command_etc("run_queue", &dump_run_queue, + "List threads in run queue", "\nLists threads in run queue", 0); + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", + 0); + if (!gSingleCore) { + add_debugger_command_etc("idle_cores", &dump_idle_cores, + "List idle cores", "\nList idle cores", 0); + } + #if SCHEDULER_TRACING add_debugger_command_etc("scheduler", &cmd_scheduler, "Analyze scheduler tracing information", @@ -1883,3 +1877,4 @@ _user_get_scheduler_mode(void) { return sCurrentModeID; } + From b41eaf299a3d74bbcd6e496dfda0d0d9d9978eac Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 26 Nov 2013 03:40:32 +0100 Subject: [PATCH 150/273] scheduler_set_thread_priority: Fix null dereference --- src/system/kernel/scheduler/scheduler.cpp | 27 ++++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index d421f81dbb..8d65740da1 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -778,11 +778,11 @@ put_back(Thread* thread) gPinnedRunQueues[pinnedCPU].PushFront(thread, get_effective_priority(thread)); } else { - int32 previougCore = thread->scheduler_data->previous_core; - ASSERT(previougCore >= 0); + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); - ASSERT(previougCore == core); - gRunQueues[previougCore].PushFront(thread, + ASSERT(previousCore == core); + gRunQueues[previousCore].PushFront(thread, get_effective_priority(thread)); } } @@ -809,9 +809,6 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->priority = priority; if (thread->state != B_THREAD_READY) { - cancel_penalty(thread); - thread->priority = priority; - if (thread->state == B_THREAD_RUNNING) update_cpu_priority(thread->cpu->cpu_num, priority); return oldPriority; @@ -821,11 +818,15 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) // a new position. bool pinned = thread->pinned_to_cpu > 0; - int32 previougCPU = thread->previous_cpu->cpu_num; - int32 previougCore = thread->scheduler_data->previous_core; - ASSERT(previougCore >= 0); + int32 previousCPU; + if (pinned) { + ASSERT(thread->previous_cpu != NULL); + previousCPU = thread->previous_cpu->cpu_num; + } + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); - SpinLocker runQueueLocker(gCoreEntries[previougCore].fQueueLock); + SpinLocker runQueueLocker(gCoreEntries[previousCore].fQueueLock); // the thread might have been already dequeued and is about to start // running once we release its scheduler_lock, in such case we can not @@ -839,9 +840,9 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->scheduler_data->enqueued = false; if (pinned) - gPinnedRunQueues[previougCPU].Remove(thread); + gPinnedRunQueues[previousCPU].Remove(thread); else - gRunQueues[previougCore].Remove(thread); + gRunQueues[previousCore].Remove(thread); runQueueLocker.Unlock(); enqueue(thread, true); From 22ae68fa692c4dd82e53001ace5f071f1b209fe8 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 27 Nov 2013 03:19:44 +0100 Subject: [PATCH 151/273] scheduler: Simplify thread priority penalties --- src/system/kernel/scheduler/scheduler.cpp | 135 ++++++------------ .../kernel/scheduler/scheduler_common.h | 13 +- 2 files changed, 52 insertions(+), 96 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 8d65740da1..cb730008c7 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -117,6 +117,7 @@ CPUEntry::CPUEntry() CoreEntry::CoreEntry() : fCPUCount(0), + fThreadCount(0), fActiveTime(0), fLoad(0) { @@ -155,9 +156,7 @@ scheduler_thread_data::Init() went_sleep = 0; went_sleep_active = 0; - - lost_cpu = false; - cpu_bound = true; + went_sleep_count = -1; previous_core = -1; enqueued = false; @@ -383,6 +382,8 @@ scheduler_dump_thread_data(Thread* thread) schedulerThreadData->went_sleep); kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", schedulerThreadData->went_sleep_active); + kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", + schedulerThreadData->went_sleep_count); kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", schedulerThreadData->previous_core); if (schedulerThreadData->previous_core > 0 @@ -626,51 +627,23 @@ thread_goes_away(Thread* thread) schedulerThreadData->went_sleep = system_time(); schedulerThreadData->went_sleep_active = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); + schedulerThreadData->went_sleep_count + = atomic_get(&gCoreEntries[smp_get_current_cpu()].fThreadCount); } static inline bool should_cancel_penalty(Thread* thread) { - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - int32 core = schedulerThreadData->previous_core; + int32 core = thread->scheduler_data->previous_core; - if (core < -1) + if (core < 0) return false; - bigtime_t now = system_time(); - bigtime_t wentSleep = schedulerThreadData->went_sleep; - - if (wentSleep < atomic_get64(&gCoreEntries[core].fReachedIdle)) - return true; - - bigtime_t startedIdle = atomic_get64(&gCoreEntries[core].fStartedIdle); - if (startedIdle != 0) { - if (wentSleep < startedIdle && now - startedIdle >= kMinimalWaitTime) - return true; - - if (wentSleep - startedIdle >= kMinimalWaitTime) - return true; - } - - if (get_effective_priority(thread) == B_LOWEST_ACTIVE_PRIORITY) - return false; - - if (wentSleep < atomic_get64(&gCoreEntries[core].fReachedBottom)) - return true; - - bigtime_t startedBottom = atomic_get64(&gCoreEntries[core].fStartedBottom); - if (gCoreEntries[core].fStartedBottom != 0) { - if (wentSleep < startedBottom - && now - startedBottom >= kMinimalWaitTime) { - return true; - } - - if (wentSleep - startedBottom >= kMinimalWaitTime) - return true; - } - - return false; + return atomic_get(&gCoreEntries[core].fThreadCount) + != thread->scheduler_data->went_sleep_count + && system_time() - thread->scheduler_data->went_sleep + > kThreadQuantum; } @@ -684,8 +657,9 @@ enqueue(Thread* thread, bool newOne) compute_thread_load(thread); scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - schedulerThreadData->cpu_bound = true; schedulerThreadData->time_left = 0; + if (!thread_is_idle_thread(thread)) + schedulerThreadData->went_sleep_count = 0; int32 threadPriority = get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); @@ -713,8 +687,11 @@ enqueue(Thread* thread, bool newOne) thread->scheduler_data->enqueued = true; if (pinned) gPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); - else + else { gRunQueues[targetCore].PushBack(thread, threadPriority); + if (!thread_is_idle_thread(thread)) + gCoreEntries[targetCore].fThreadList.Insert(thread->scheduler_data); + } runQueueLocker.Unlock(); // notify listeners @@ -766,6 +743,7 @@ static inline void put_back(Thread* thread) { compute_thread_load(thread); + thread->scheduler_data->went_sleep_count = -1; int32 core = gCPUToCore[smp_get_current_cpu()]; @@ -841,8 +819,15 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->scheduler_data->enqueued = false; if (pinned) gPinnedRunQueues[previousCPU].Remove(thread); - else + else { gRunQueues[previousCore].Remove(thread); + + ASSERT(thread->scheduler_data->went_sleep_count < 1); + if (thread->scheduler_data->went_sleep_count == 0) { + gCoreEntries[previousCore].fThreadList.Remove( + thread->scheduler_data); + } + } runQueueLocker.Unlock(); enqueue(thread, true); @@ -857,7 +842,6 @@ reschedule_needed() { // This function is called as a result of either the timer event set by the // scheduler or an incoming ICI. Make sure the reschedule() is invoked. - thread_get_current_thread()->scheduler_data->lost_cpu = true; get_cpu_struct()->invoke_scheduler = true; } @@ -988,6 +972,17 @@ choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) sharedThread->scheduler_data->enqueued = false; gRunQueues[thisCore].Remove(sharedThread); + if (thread_is_idle_thread(sharedThread) + || gCoreEntries[thisCore].fThreadList.Head() + == sharedThread->scheduler_data) { + atomic_add(&gCoreEntries[thisCore].fThreadCount, 1); + } + + if (sharedThread->scheduler_data->went_sleep_count == 0) { + gCoreEntries[thisCore].fThreadList.Remove( + sharedThread->scheduler_data); + } + return sharedThread; } @@ -1005,19 +1000,6 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) bigtime_t now = system_time(); bigtime_t usedTime = now - oldThread->scheduler_data->quantum_start; - if (thread_is_idle_thread(oldThread) && usedTime >= kMinimalWaitTime) { - atomic_set64(&gCoreEntries[thisCore].fReachedBottom, - now - kMinimalWaitTime); - atomic_set64(&gCoreEntries[thisCore].fReachedIdle, - now - kMinimalWaitTime); - } - - if (get_effective_priority(oldThread) == B_LOWEST_ACTIVE_PRIORITY - && usedTime >= kMinimalWaitTime) { - atomic_set64(&gCoreEntries[thisCore].fReachedBottom, - now - kMinimalWaitTime); - } - if (!thread_is_idle_thread(oldThread)) { bigtime_t active = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) @@ -1030,26 +1012,14 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) atomic_add64(&gCoreEntries[thisCore].fActiveTime, active); } + compute_thread_load(oldThread); + compute_thread_load(nextThread); if (!gSingleCore && !gCPU[smp_get_current_cpu()].disabled) compute_cpu_load(smp_get_current_cpu()); int32 oldPriority = get_effective_priority(oldThread); int32 nextPriority = get_effective_priority(nextThread); - if (thread_is_idle_thread(nextThread)) { - if (!thread_is_idle_thread(oldThread)) - atomic_set64(&gCoreEntries[thisCore].fStartedIdle, now); - if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - atomic_set64(&gCoreEntries[thisCore].fStartedBottom, now); - } else if (nextPriority == B_LOWEST_ACTIVE_PRIORITY) { - atomic_set64(&gCoreEntries[thisCore].fStartedIdle, 0); - if (oldPriority > B_LOWEST_ACTIVE_PRIORITY) - atomic_set64(&gCoreEntries[thisCore].fStartedBottom, now); - } else { - atomic_set64(&gCoreEntries[thisCore].fStartedBottom, 0); - atomic_set64(&gCoreEntries[thisCore].fStartedIdle, 0); - } - if (!thread_is_idle_thread(nextThread)) { oldThread->cpu->last_kernel_time = nextThread->kernel_time; oldThread->cpu->last_user_time = nextThread->user_time; @@ -1200,13 +1170,9 @@ reschedule(void) case B_THREAD_READY: enqueueOldThread = true; - if (!schedulerOldThreadData->lost_cpu) - schedulerOldThreadData->cpu_bound = false; - if (quantum_ended(oldThread, oldThread->cpu->preempted, oldThread->has_yielded)) { - if (schedulerOldThreadData->cpu_bound) - increase_penalty(oldThread); + increase_penalty(oldThread); TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, get_effective_priority(oldThread)); @@ -1217,11 +1183,6 @@ reschedule(void) putOldThreadAtBack = false; } - break; - case B_THREAD_SUSPENDED: - increase_penalty(oldThread); - thread_goes_away(oldThread); - TRACE("reschedule(): suspending thread %ld\n", oldThread->id); break; case THREAD_STATE_FREE_ON_RESCHED: break; @@ -1234,7 +1195,6 @@ reschedule(void) } oldThread->has_yielded = false; - schedulerOldThreadData->lost_cpu = false; // select thread with the biggest priority and enqueue back the old thread Thread* nextThread; @@ -1284,8 +1244,6 @@ reschedule(void) ASSERT(nextThread->scheduler_data->previous_core == thisCore); - compute_thread_load(nextThread); - // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); @@ -1296,6 +1254,9 @@ reschedule(void) nextThread->scheduler_data->last_interrupt_time = gCPU[thisCPU].interrupt_time; + if (!thread_is_idle_thread(nextThread)) + update_cpu_performance(nextThread, thisCore); + if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; if (!oldThread->cpu->preempted) @@ -1306,8 +1267,6 @@ reschedule(void) bigtime_t quantum = compute_quantum(oldThread); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); - - update_cpu_performance(nextThread, thisCore); } else { nextThread->scheduler_data->quantum_start = system_time(); @@ -1775,10 +1734,10 @@ scheduler_init(void) add_debugger_command_etc("run_queue", &dump_run_queue, "List threads in run queue", "\nLists threads in run queue", 0); - add_debugger_command_etc("cpu_heap", &dump_cpu_heap, - "List CPUs in CPU priority heap", "\nList CPUs in CPU priority heap", - 0); if (!gSingleCore) { + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", + "\nList CPUs in CPU priority heap", 0); add_debugger_command_etc("idle_cores", &dump_idle_cores, "List idle cores", "\nList idle cores", 0); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index dc07cf5bac..2ad5ecf4c3 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -81,10 +81,8 @@ struct CoreEntry : public MinMaxHeapLinkImpl, spinlock fCPULock; spinlock fQueueLock; - bigtime_t fStartedBottom; - bigtime_t fReachedBottom; - bigtime_t fStartedIdle; - bigtime_t fReachedIdle; + int32 fThreadCount; + DoublyLinkedList fThreadList; bigtime_t fActiveTime; @@ -146,16 +144,14 @@ extern int32* gCPUToPackage; } // namespace Scheduler -struct scheduler_thread_data { +struct scheduler_thread_data : + public DoublyLinkedListLinkImpl { inline scheduler_thread_data(); void Init(); int32 priority_penalty; int32 additional_penalty; - bool lost_cpu; - bool cpu_bound; - bigtime_t time_left; bigtime_t stolen_time; bigtime_t quantum_start; @@ -167,6 +163,7 @@ struct scheduler_thread_data { bigtime_t went_sleep; bigtime_t went_sleep_active; + int32 went_sleep_count; int32 previous_core; From 28da7e985fd8463ec31ce9ebbb4b9610b361f8be Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 27 Nov 2013 03:36:06 +0100 Subject: [PATCH 152/273] scheduler: Improve debug commands --- src/system/kernel/scheduler/scheduler.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index cb730008c7..c5e48955e4 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -224,15 +224,11 @@ static int dump_run_queue(int argc, char **argv) { int32 cpuCount = smp_get_num_cpus(); - int32 coreCount = 0; - for (int32 i = 0; i < cpuCount; i++) { - if (gCPU[i].topology_id[CPU_TOPOLOGY_SMT] == 0) - gCPUToCore[i] = coreCount++; - } + int32 coreCount = gRunQueueCount; ThreadRunQueue::ConstIterator iterator; for (int32 i = 0; i < coreCount; i++) { - kprintf("\nCore %" B_PRId32 " run queue:\n", i); + kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); iterator = gRunQueues[i].GetConstIterator(); dump_queue(iterator); } @@ -240,7 +236,7 @@ dump_run_queue(int argc, char **argv) for (int32 i = 0; i < cpuCount; i++) { iterator = gPinnedRunQueues[i].GetConstIterator(); - if (iterator.HasNext()) { + if (iterator.HasNext() && !thread_is_idle_thread(iterator.Next())) { kprintf("\nCPU %" B_PRId32 " run queue:\n", i); dump_queue(iterator); } @@ -251,7 +247,7 @@ dump_run_queue(int argc, char **argv) static void -dump_heap(CPUHeap* heap) +dump_cpu_load_heap(CPUHeap* heap) { kprintf("cpu priority load\n"); CPUEntry* entry = heap->PeekMinimum(); @@ -305,14 +301,16 @@ dump_core_load_heap(CoreLoadHeap* heap) static int dump_cpu_heap(int argc, char** argv) { - kprintf("\ncore load\n"); + kprintf("core load\n"); dump_core_load_heap(gCoreLoadHeap); - kprintf("---------\n"); dump_core_load_heap(gCoreHighLoadHeap); for (int32 i = 0; i < gRunQueueCount; i++) { + if (gCoreEntries[i].fCPUCount < 2) + continue; + kprintf("\nCore %" B_PRId32 " heap:\n", i); - dump_heap(&gCPUPriorityHeaps[i]); + dump_cpu_load_heap(&gCPUPriorityHeaps[i]); } return 0; From 7d7dc357bf0966b28dce9e0f08bff04353be7098 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 27 Nov 2013 03:50:43 +0100 Subject: [PATCH 153/273] scheduler: Move simple inline functions to scheduler_common.h --- src/system/kernel/scheduler/scheduler.cpp | 54 +++---------------- .../kernel/scheduler/scheduler_common.h | 44 +++++++++++++-- 2 files changed, 48 insertions(+), 50 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index c5e48955e4..a265671483 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -61,15 +61,15 @@ CoreEntry* gCoreEntries; CoreLoadHeap* gCoreLoadHeap; CoreLoadHeap* gCoreHighLoadHeap; rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; +int32 gCoreCount; PackageEntry* gPackageEntries; IdlePackageList* gIdlePackageList; -spinlock gIdlePackageLock; -int32 gPackageCount = B_SPINLOCK_INITIALIZER; +spinlock gIdlePackageLock = B_SPINLOCK_INITIALIZER; +int32 gPackageCount; ThreadRunQueue* gRunQueues; ThreadRunQueue* gPinnedRunQueues; -int32 gRunQueueCount; int32* gCPUToCore; int32* gCPUToPackage; @@ -163,46 +163,6 @@ scheduler_thread_data::Init() } -static inline int -get_minimal_priority(Thread* thread) -{ - return max_c(min_c(thread->priority, 25) / 5, 1); -} - - -static inline int32 -get_thread_penalty(Thread* thread) -{ - int32 penalty = thread->scheduler_data->priority_penalty; - - const int kMinimalPriority = get_minimal_priority(thread); - if (kMinimalPriority > 0) { - penalty - += thread->scheduler_data->additional_penalty % kMinimalPriority; - } - - return penalty; -} - - -static inline int32 -get_effective_priority(Thread* thread) -{ - if (thread->priority == B_IDLE_PRIORITY) - return thread->priority; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return thread->priority; - - int32 effectivePriority = thread->priority; - effectivePriority -= get_thread_penalty(thread); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; -} - - static void dump_queue(ThreadRunQueue::ConstIterator& iterator) { @@ -224,7 +184,7 @@ static int dump_run_queue(int argc, char **argv) { int32 cpuCount = smp_get_num_cpus(); - int32 coreCount = gRunQueueCount; + int32 coreCount = gCoreCount; ThreadRunQueue::ConstIterator iterator; for (int32 i = 0; i < coreCount; i++) { @@ -305,7 +265,7 @@ dump_cpu_heap(int argc, char** argv) dump_core_load_heap(gCoreLoadHeap); dump_core_load_heap(gCoreHighLoadHeap); - for (int32 i = 0; i < gRunQueueCount; i++) { + for (int32 i = 0; i < gCoreCount; i++) { if (gCoreEntries[i].fCPUCount < 2) continue; @@ -559,7 +519,7 @@ choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) targetCPU = choose_cpu(targetCore); } - ASSERT(targetCore >= 0 && targetCore < gRunQueueCount); + ASSERT(targetCore >= 0 && targetCore < gCoreCount); ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); } @@ -1598,7 +1558,7 @@ init() packageCount); if (result != B_OK) return result; - gRunQueueCount = coreCount; + gCoreCount = coreCount; gSingleCore = coreCount == 1; gPackageCount = packageCount; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 2ad5ecf4c3..c8037758a0 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -33,8 +33,6 @@ const bigtime_t kThreadQuantum = 1000; const bigtime_t kMinThreadQuantum = 3000; const bigtime_t kMaxThreadQuantum = 10000; -const bigtime_t kMinimalWaitTime = kThreadQuantum / 4; - const bigtime_t kCacheExpire = 100000; const int kLowLoad = kMaxLoad * 20 / 100; @@ -94,6 +92,7 @@ extern CoreEntry* gCoreEntries; extern CoreLoadHeap* gCoreLoadHeap; extern CoreLoadHeap* gCoreHighLoadHeap; extern rw_spinlock gCoreHeapsLock; +extern int32 gCoreCount; // gPackageEntries are used to decide which core should be woken up from the // idle state. When aiming for performance we should use as many packages as @@ -132,7 +131,6 @@ typedef RunQueue CACHE_LINE_ALIGN extern ThreadRunQueue* gRunQueues; extern ThreadRunQueue* gPinnedRunQueues; -extern int32 gRunQueueCount; // Since CPU IDs used internally by the kernel bear no relation to the actual // CPU topology the following arrays are used to efficiently get the core @@ -178,4 +176,44 @@ get_core_load(struct Scheduler::CoreEntry* core) } +static inline int32 +get_minimal_priority(Thread* thread) +{ + return max_c(min_c(thread->priority, 25) / 5, 1); +} + + +static inline int32 +get_thread_penalty(Thread* thread) +{ + int32 penalty = thread->scheduler_data->priority_penalty; + + const int kMinimalPriority = get_minimal_priority(thread); + if (kMinimalPriority > 0) { + penalty + += thread->scheduler_data->additional_penalty % kMinimalPriority; + } + + return penalty; +} + + +static inline int32 +get_effective_priority(Thread* thread) +{ + if (thread->priority == B_IDLE_PRIORITY) + return thread->priority; + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return thread->priority; + + int32 effectivePriority = thread->priority; + effectivePriority -= get_thread_penalty(thread); + + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return effectivePriority; +} + + #endif // KERNEL_SCHEDULER_COMMON_H From 9caf7f4fb95b33daa9e2caf2267c2636ba24ce49 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 27 Nov 2013 04:15:36 +0100 Subject: [PATCH 154/273] scheduler: Update estimate_max_scheduling_latency() --- src/system/kernel/scheduler/scheduler.cpp | 23 +++++++++++-------- .../kernel/scheduler/scheduler_common.h | 2 -- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index a265671483..6f105f630a 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -1768,18 +1769,20 @@ _user_estimate_max_scheduling_latency(thread_id id) } BReference threadReference(thread, true); - // TODO: This is probably meant to be called periodically to return the - // current estimate depending on the system usage; we return fixed estimates - // per thread priority, though. + int32 core = thread->scheduler_data->previous_core; + if (core == -1) + core = get_random() % gCoreCount; - if (thread->priority >= B_REAL_TIME_DISPLAY_PRIORITY) - return kMinThreadQuantum / 4; - if (thread->priority >= B_DISPLAY_PRIORITY) - return kMinThreadQuantum; - if (thread->priority < B_NORMAL_PRIORITY) - return 2 * kMaxThreadQuantum; + int32 threadCount = gCoreEntries[core].fThreadCount; + if (gCoreEntries[core].fCPUCount > 0) + threadCount /= gCoreEntries[core].fCPUCount; - return 2 * kMinThreadQuantum; + if (get_effective_priority(thread) > 0) { + threadCount -= threadCount * THREAD_MAX_SET_PRIORITY + / get_effective_priority(thread); + } + + return max_c(threadCount * kThreadQuantum, kThreadQuantum / 5); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index c8037758a0..1fdb26f8f1 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -30,8 +30,6 @@ namespace Scheduler { const bigtime_t kThreadQuantum = 1000; -const bigtime_t kMinThreadQuantum = 3000; -const bigtime_t kMaxThreadQuantum = 10000; const bigtime_t kCacheExpire = 100000; From 87115715b40c394b08de08b6709863eb257b020d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 27 Nov 2013 04:57:26 +0100 Subject: [PATCH 155/273] scheduler: Protect package data with rw_spinlock --- src/system/kernel/scheduler/low_latency.cpp | 4 ++-- src/system/kernel/scheduler/power_saving.cpp | 4 ++-- src/system/kernel/scheduler/scheduler.cpp | 18 ++++++------------ src/system/kernel/scheduler/scheduler_common.h | 4 ++-- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 832b7ab1d3..a3eb28be4a 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -63,7 +63,7 @@ choose_core(Thread* thread) { CoreEntry* entry = NULL; - SpinLocker locker(gIdlePackageLock); + ReadSpinLocker locker(gIdlePackageLock); // wake new package PackageEntry* package = gIdlePackageList->Last(); if (package == NULL) { @@ -73,7 +73,7 @@ choose_core(Thread* thread) locker.Unlock(); if (package != NULL) { - SpinLocker _(package->fCoreLock); + ReadSpinLocker _(package->fCoreLock); entry = package->fIdleCores.Last(); } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 63ad0e3c23..75d754f178 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -89,12 +89,12 @@ choose_idle_core(void) } if (current == NULL) { - SpinLocker _(gIdlePackageLock); + ReadSpinLocker _(gIdlePackageLock); current = gIdlePackageList->Last(); } if (current != NULL) { - SpinLocker _(current->fCoreLock); + ReadSpinLocker _(current->fCoreLock); return current->fIdleCores.Last(); } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 6f105f630a..ede18e6a26 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -66,7 +66,7 @@ int32 gCoreCount; PackageEntry* gPackageEntries; IdlePackageList* gIdlePackageList; -spinlock gIdlePackageLock = B_SPINLOCK_INITIALIZER; +rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; int32 gPackageCount; ThreadRunQueue* gRunQueues; @@ -132,7 +132,7 @@ PackageEntry::PackageEntry() fIdleCoreCount(0), fCoreCount(0) { - B_INITIALIZE_SPINLOCK(&fCoreLock); + B_INITIALIZE_RW_SPINLOCK(&fCoreLock); } @@ -457,7 +457,7 @@ update_cpu_priority(int32 cpu, int32 priority) int32 package = gCPUToPackage[cpu]; PackageEntry* packageEntry = &gPackageEntries[package]; if (maxPriority == B_IDLE_PRIORITY) { - SpinLocker _(packageEntry->fCoreLock); + WriteSpinLocker _(packageEntry->fCoreLock); // core goes idle ASSERT(packageEntry->fIdleCoreCount >= 0); @@ -468,11 +468,11 @@ update_cpu_priority(int32 cpu, int32 priority) if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { // package goes idle - SpinLocker _(gIdlePackageLock); + WriteSpinLocker _(gIdlePackageLock); gIdlePackageList->Add(packageEntry); } } else if (corePriority == B_IDLE_PRIORITY) { - SpinLocker _(packageEntry->fCoreLock); + WriteSpinLocker _(packageEntry->fCoreLock); // core wakes up ASSERT(packageEntry->fIdleCoreCount > 0); @@ -483,7 +483,7 @@ update_cpu_priority(int32 cpu, int32 priority) if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { // package wakes up - SpinLocker _(gIdlePackageLock); + WriteSpinLocker _(gIdlePackageLock); gIdlePackageList->Remove(packageEntry); } } @@ -956,9 +956,6 @@ choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) static inline void track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) { - bigtime_t now = system_time(); - bigtime_t usedTime = now - oldThread->scheduler_data->quantum_start; - if (!thread_is_idle_thread(oldThread)) { bigtime_t active = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) @@ -976,9 +973,6 @@ track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) if (!gSingleCore && !gCPU[smp_get_current_cpu()].disabled) compute_cpu_load(smp_get_current_cpu()); - int32 oldPriority = get_effective_priority(oldThread); - int32 nextPriority = get_effective_priority(nextThread); - if (!thread_is_idle_thread(nextThread)) { oldThread->cpu->last_kernel_time = nextThread->kernel_time; oldThread->cpu->last_user_time = nextThread->user_time; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 1fdb26f8f1..f379ba9e36 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -106,7 +106,7 @@ struct PackageEntry : public DoublyLinkedListLinkImpl { int32 fPackageID; - spinlock fCoreLock; + rw_spinlock fCoreLock; DoublyLinkedList fIdleCores; int32 fIdleCoreCount; @@ -117,7 +117,7 @@ typedef DoublyLinkedList IdlePackageList; extern PackageEntry* gPackageEntries; extern IdlePackageList* gIdlePackageList; -extern spinlock gIdlePackageLock; +extern rw_spinlock gIdlePackageLock; extern int32 gPackageCount; // The run queues. Holds the threads ready to run ordered by priority. From 286b341a400e8d12060a8be52214618b8f02df87 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 28 Nov 2013 14:03:57 +0100 Subject: [PATCH 156/273] kernel: Merge two occurences of thread resume code --- headers/private/kernel/kscheduler.h | 2 + src/system/kernel/scheduler/scheduler.cpp | 91 +++++++++++++++-------- src/system/kernel/thread.cpp | 18 +---- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index 2d8222ebed..cbad85d9e2 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -83,6 +83,8 @@ status_t scheduler_set_operation_mode(scheduler_mode mode); */ void scheduler_dump_thread_data(Thread* thread); +void scheduler_new_thread_entry(Thread* thread); + void scheduler_set_cpu_enabled(int32 cpu, bool enabled); void scheduler_add_listener(struct SchedulerListener* listener); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ede18e6a26..b1bcef7d15 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1007,6 +1007,58 @@ update_cpu_performance(Thread* thread, int32 thisCore) } +static inline void +stop_cpu_timers(Thread* fromThread, Thread* toThread) +{ + SpinLocker teamLocker(&fromThread->team->time_lock); + SpinLocker threadLocker(&fromThread->time_lock); + + if (fromThread->HasActiveCPUTimeUserTimers() + || fromThread->team->HasActiveCPUTimeUserTimers()) { + user_timer_stop_cpu_timers(fromThread, toThread); + } +} + + +static inline void +continue_cpu_timers(Thread* thread, cpu_ent* cpu) +{ + SpinLocker teamLocker(&thread->team->time_lock); + SpinLocker threadLocker(&thread->time_lock); + + if (thread->HasActiveCPUTimeUserTimers() + || thread->team->HasActiveCPUTimeUserTimers()) { + user_timer_continue_cpu_timers(thread, cpu->previous_thread); + } +} + + +static void +thread_resumes(Thread* thread) +{ + cpu_ent* cpu = thread->cpu; + + release_spinlock(&cpu->previous_thread->scheduler_lock); + + // continue CPU time based user timers + continue_cpu_timers(thread, cpu); + + // notify the user debugger code + if ((thread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) + user_debug_thread_scheduled(thread); +} + + +void +scheduler_new_thread_entry(Thread* thread) +{ + thread_resumes(thread); + + SpinLocker locker(thread->time_lock); + thread->last_time = system_time(); +} + + /*! Switches the currently running thread. This is a service function for scheduler implementations. @@ -1022,14 +1074,7 @@ switch_thread(Thread* fromThread, Thread* toThread) user_debug_thread_unscheduled(fromThread); // stop CPU time based user timers - acquire_spinlock(&fromThread->team->time_lock); - acquire_spinlock(&fromThread->time_lock); - if (fromThread->HasActiveCPUTimeUserTimers() - || fromThread->team->HasActiveCPUTimeUserTimers()) { - user_timer_stop_cpu_timers(fromThread, toThread); - } - release_spinlock(&fromThread->time_lock); - release_spinlock(&fromThread->team->time_lock); + stop_cpu_timers(fromThread, toThread); // update CPU and Thread structures and perform the context switch cpu_ent* cpu = fromThread->cpu; @@ -1041,25 +1086,10 @@ switch_thread(Thread* fromThread, Thread* toThread) arch_thread_set_current_thread(toThread); arch_thread_context_switch(fromThread, toThread); - release_spinlock(&fromThread->cpu->previous_thread->scheduler_lock); - // The use of fromThread below looks weird, but is correct. fromThread had // been unscheduled earlier, but is back now. For a thread scheduled the // first time the same is done in thread.cpp:common_thread_entry(). - - // continue CPU time based user timers - acquire_spinlock(&fromThread->team->time_lock); - acquire_spinlock(&fromThread->time_lock); - if (fromThread->HasActiveCPUTimeUserTimers() - || fromThread->team->HasActiveCPUTimeUserTimers()) { - user_timer_continue_cpu_timers(fromThread, cpu->previous_thread); - } - release_spinlock(&fromThread->time_lock); - release_spinlock(&fromThread->team->time_lock); - - // notify the user debugger code - if ((fromThread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) - user_debug_thread_scheduled(fromThread); + thread_resumes(fromThread); } @@ -1068,28 +1098,25 @@ update_thread_times(Thread* oldThread, Thread* nextThread) { bigtime_t now = system_time(); if (oldThread == nextThread) { - acquire_spinlock(&oldThread->time_lock); + SpinLocker _(oldThread->time_lock); oldThread->kernel_time += now - oldThread->last_time; oldThread->last_time = now; - release_spinlock(&oldThread->time_lock); } else { - acquire_spinlock(&oldThread->time_lock); + SpinLocker locker(oldThread->time_lock); oldThread->kernel_time += now - oldThread->last_time; oldThread->last_time = 0; - release_spinlock(&oldThread->time_lock); + locker.Unlock(); - acquire_spinlock(&nextThread->time_lock); + locker.SetTo(nextThread->time_lock, false); nextThread->last_time = now; - release_spinlock(&nextThread->time_lock); } // If the old thread's team has user time timers, check them now. Team* team = oldThread->team; - acquire_spinlock(&team->time_lock); + SpinLocker _(team->time_lock); if (team->HasActiveUserTimeUserTimers()) user_timer_check_team_user_timers(team); - release_spinlock(&team->time_lock); } diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index ca66a46140..f3474b58ca 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -698,25 +698,9 @@ common_thread_entry(void* _args) // The thread is new and has been scheduled the first time. - // start CPU time based user timers - acquire_spinlock(&thread->team->time_lock); - acquire_spinlock(&thread->time_lock); - if (thread->HasActiveCPUTimeUserTimers() - || thread->team->HasActiveCPUTimeUserTimers()) { - user_timer_continue_cpu_timers(thread, thread->cpu->previous_thread); - } - - // start tracking time - thread->last_time = system_time(); - release_spinlock(&thread->time_lock); - release_spinlock(&thread->team->time_lock); - - // notify the user debugger code - if ((thread->flags & THREAD_FLAGS_DEBUGGER_INSTALLED) != 0) - user_debug_thread_scheduled(thread); + scheduler_new_thread_entry(thread); // unlock the scheduler lock and enable interrupts - release_spinlock(&thread->cpu->previous_thread->scheduler_lock); release_spinlock(&thread->scheduler_lock); enable_interrupts(); From f9ee217ad6c59122fefd39a445c99f2c0fb2af1e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 28 Nov 2013 16:33:50 +0100 Subject: [PATCH 157/273] scheduler: Migrate threads less often in power saving mode --- src/system/kernel/scheduler/low_latency.cpp | 3 - src/system/kernel/scheduler/power_saving.cpp | 60 ++++++++----------- src/system/kernel/scheduler/scheduler.cpp | 19 ++++-- .../kernel/scheduler/scheduler_common.h | 2 + 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index a3eb28be4a..3e4365768b 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -30,9 +30,6 @@ has_cache_expired(Thread* thread) { ASSERT(!gSingleCore); - if (thread_is_idle_thread(thread)) - return false; - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; ASSERT(schedulerThreadData->previous_core >= 0); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 75d754f178..aa960e49cd 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -36,9 +36,6 @@ has_cache_expired(Thread* thread) { ASSERT(!gSingleCore); - if (thread_is_idle_thread(thread)) - return false; - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; ASSERT(schedulerThreadData->previous_core >= 0); @@ -46,19 +43,6 @@ has_cache_expired(Thread* thread) } -static bool -try_small_task_packing(Thread* thread) -{ - ReadSpinLocker locker(gCoreHeapsLock); - - int32 core = sSmallTaskCore; - return (core == -1 && gCoreLoadHeap->PeekMaximum() != NULL) - || (core != -1 - && get_core_load(&gCoreEntries[core]) + thread->scheduler_data->load - < kHighLoad); -} - - static int32 choose_small_task_core(void) { @@ -107,24 +91,27 @@ choose_core(Thread* thread) { CoreEntry* entry; - if (try_small_task_packing(thread)) { - // try to pack all threads on one core - entry = &gCoreEntries[choose_small_task_core()]; + int32 core = -1; + // try to pack all threads on one core + core = choose_small_task_core(); + + if (core != -1 + && get_core_load(&gCoreEntries[core]) + thread->scheduler_data->load + < kHighLoad) { + entry = &gCoreEntries[core]; } else { ReadSpinLocker coreLocker(gCoreHeapsLock); - if (gCoreLoadHeap->PeekMinimum() != NULL) { - // run immediately on already woken core - entry = gCoreLoadHeap->PeekMinimum(); - } else { + // run immediately on already woken core + entry = gCoreLoadHeap->PeekMinimum(); + if (entry == NULL) { coreLocker.Unlock(); entry = choose_idle_core(); - coreLocker.Lock(); - if (entry == NULL) - entry = gCoreLoadHeap->PeekMinimum(); - if (entry == NULL) + if (entry == NULL) { + coreLocker.Lock(); entry = gCoreHighLoadHeap->PeekMinimum(); + } } } @@ -138,9 +125,6 @@ should_rebalance(Thread* thread) { ASSERT(!gSingleCore); - if (thread_is_idle_thread(thread)) - return false; - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; ASSERT(schedulerThreadData->previous_core >= 0); @@ -151,13 +135,16 @@ should_rebalance(Thread* thread) if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { - if (coreLoad - schedulerThreadData->load < kHighLoad) - return true; - + sSmallTaskCore = -1; choose_small_task_core(); + if (schedulerThreadData->load > coreLoad / 3) + return false; return coreLoad > kVeryHighLoad; } + if (schedulerThreadData->load >= coreLoad / 2) + return false; + CoreEntry* other = gCoreLoadHeap->PeekMaximum(); if (other == NULL) other = gCoreHighLoadHeap->PeekMinimum(); @@ -165,10 +152,15 @@ should_rebalance(Thread* thread) return coreLoad - get_core_load(other) >= kLoadDifference / 2; } + if (coreLoad >= kMediumLoad) + return false; + int32 smallTaskCore = choose_small_task_core(); if (smallTaskCore == -1) return false; - return smallTaskCore != core; + return smallTaskCore != core + && get_core_load(&gCoreEntries[smallTaskCore]) + + thread->scheduler_data->load < kHighLoad; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index b1bcef7d15..8fb6cf970f 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -120,7 +120,8 @@ CoreEntry::CoreEntry() fCPUCount(0), fThreadCount(0), fActiveTime(0), - fLoad(0) + fLoad(0), + fHighLoad(false) { B_INITIALIZE_SPINLOCK(&fCPULock); B_INITIALIZE_SPINLOCK(&fQueueLock); @@ -264,6 +265,7 @@ dump_cpu_heap(int argc, char** argv) { kprintf("core load\n"); dump_core_load_heap(gCoreLoadHeap); + kprintf("\n"); dump_core_load_heap(gCoreHighLoadHeap); for (int32 i = 0; i < gCoreCount; i++) { @@ -376,23 +378,32 @@ update_load_heaps(int32 core) return; if (newKey > kHighLoad) { - if (oldKey <= kHighLoad) { + if (!entry->fHighLoad) { gCoreLoadHeap->ModifyKey(entry, -1); ASSERT(gCoreLoadHeap->PeekMinimum() == entry); gCoreLoadHeap->RemoveMinimum(); gCoreHighLoadHeap->Insert(entry, newKey); + + entry->fHighLoad = true; } else gCoreHighLoadHeap->ModifyKey(entry, newKey); - } else { - if (oldKey > kHighLoad) { + } else if (newKey < kMediumLoad) { + if (entry->fHighLoad) { gCoreHighLoadHeap->ModifyKey(entry, -1); ASSERT(gCoreHighLoadHeap->PeekMinimum() == entry); gCoreHighLoadHeap->RemoveMinimum(); gCoreLoadHeap->Insert(entry, newKey); + + entry->fHighLoad = false; } else gCoreLoadHeap->ModifyKey(entry, newKey); + } else { + if (entry->fHighLoad) + gCoreHighLoadHeap->ModifyKey(entry, newKey); + else + gCoreLoadHeap->ModifyKey(entry, newKey); } } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index f379ba9e36..e92b3a6763 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -36,6 +36,7 @@ const bigtime_t kCacheExpire = 100000; const int kLowLoad = kMaxLoad * 20 / 100; const int kTargetLoad = kMaxLoad * 55 / 100; const int kHighLoad = kMaxLoad * 70 / 100; +const int kMediumLoad = (kHighLoad + kTargetLoad) / 2; const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; const int kLoadDifference = kMaxLoad * 20 / 100; @@ -83,6 +84,7 @@ struct CoreEntry : public MinMaxHeapLinkImpl, bigtime_t fActiveTime; int32 fLoad; + bool fHighLoad; } CACHE_LINE_ALIGN; typedef MinMaxHeap CoreLoadHeap; From f2243876dfa5c8543ca031d685a9d403d292757a Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 28 Nov 2013 17:53:19 +0100 Subject: [PATCH 158/273] scheduler: Remove some unnecessary checks against idle threads --- src/system/kernel/scheduler/scheduler.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 8fb6cf970f..343c5ee693 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -587,9 +587,6 @@ compute_thread_load(Thread* thread) static inline void thread_goes_away(Thread* thread) { - if (thread_is_idle_thread(thread)) - return; - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; schedulerThreadData->last_interrupt_time = 0; @@ -628,8 +625,7 @@ enqueue(Thread* thread, bool newOne) scheduler_thread_data* schedulerThreadData = thread->scheduler_data; schedulerThreadData->time_left = 0; - if (!thread_is_idle_thread(thread)) - schedulerThreadData->went_sleep_count = 0; + schedulerThreadData->went_sleep_count = 0; int32 threadPriority = get_effective_priority(thread); T(EnqueueThread(thread, threadPriority)); @@ -659,8 +655,7 @@ enqueue(Thread* thread, bool newOne) gPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); else { gRunQueues[targetCore].PushBack(thread, threadPriority); - if (!thread_is_idle_thread(thread)) - gCoreEntries[targetCore].fThreadList.Insert(thread->scheduler_data); + gCoreEntries[targetCore].fThreadList.Insert(thread->scheduler_data); } runQueueLocker.Unlock(); From 7b4befcd47d4a7ee56d50b40a41cf79aa23cf7c6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 28 Nov 2013 21:31:48 +0100 Subject: [PATCH 159/273] scheduler: Introduce upper bound on latency --- src/system/kernel/scheduler/scheduler.cpp | 39 ++++++++++++------- .../kernel/scheduler/scheduler_common.h | 5 ++- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 343c5ee693..ee659d1732 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -118,6 +118,7 @@ CPUEntry::CPUEntry() CoreEntry::CoreEntry() : fCPUCount(0), + fStarvationCounter(0), fThreadCount(0), fActiveTime(0), fLoad(0), @@ -595,7 +596,7 @@ thread_goes_away(Thread* thread) schedulerThreadData->went_sleep_active = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); schedulerThreadData->went_sleep_count - = atomic_get(&gCoreEntries[smp_get_current_cpu()].fThreadCount); + = atomic_get(&gCoreEntries[smp_get_current_cpu()].fStarvationCounter); } @@ -607,7 +608,7 @@ should_cancel_penalty(Thread* thread) if (core < 0) return false; - return atomic_get(&gCoreEntries[core].fThreadCount) + return atomic_get(&gCoreEntries[core].fStarvationCounter) != thread->scheduler_data->went_sleep_count && system_time() - thread->scheduler_data->went_sleep > kThreadQuantum; @@ -656,6 +657,8 @@ enqueue(Thread* thread, bool newOne) else { gRunQueues[targetCore].PushBack(thread, threadPriority); gCoreEntries[targetCore].fThreadList.Insert(thread->scheduler_data); + + atomic_add(&gCoreEntries[targetCore].fThreadCount, 1); } runQueueLocker.Unlock(); @@ -721,12 +724,10 @@ put_back(Thread* thread) gPinnedRunQueues[pinnedCPU].PushFront(thread, get_effective_priority(thread)); } else { - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); + ASSERT(thread->scheduler_data->previous_core == core); - ASSERT(previousCore == core); - gRunQueues[previousCore].PushFront(thread, - get_effective_priority(thread)); + gRunQueues[core].PushFront(thread, get_effective_priority(thread)); + atomic_add(&gCoreEntries[core].fThreadCount, 1); } } @@ -792,6 +793,8 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) gCoreEntries[previousCore].fThreadList.Remove( thread->scheduler_data); } + + atomic_add(&gCoreEntries[previousCore].fThreadCount, -1); } runQueueLocker.Unlock(); @@ -872,12 +875,12 @@ get_base_quantum(Thread* thread) if (priority >= B_URGENT_DISPLAY_PRIORITY) return kThreadQuantum; if (priority > B_NORMAL_PRIORITY) { - return quantum_linear_interpolation(kThreadQuantum * 4, + return quantum_linear_interpolation(kThreadQuantum * 2, kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, priority); } - return quantum_linear_interpolation(kThreadQuantum * 64, - kThreadQuantum * 4, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); + return quantum_linear_interpolation(kThreadQuantum * 30, + kThreadQuantum * 2, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); } @@ -895,9 +898,17 @@ compute_quantum(Thread* thread) quantum += schedulerThreadData->stolen_time; schedulerThreadData->stolen_time = 0; + ASSERT(schedulerThreadData->previous_core + == gCPUToCore[smp_get_current_cpu()]); + CoreEntry* core = &gCoreEntries[schedulerThreadData->previous_core]; + int32 threadCount = (core->fThreadCount + 1) / core->fCPUCount; + if (threadCount > 1) { + quantum = max_c(min_c(kMaximumLatency / threadCount, quantum), + kThreadQuantum / 3); + } + schedulerThreadData->time_left = quantum; schedulerThreadData->quantum_start = system_time(); - return quantum; } @@ -940,7 +951,7 @@ choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) if (thread_is_idle_thread(sharedThread) || gCoreEntries[thisCore].fThreadList.Head() == sharedThread->scheduler_data) { - atomic_add(&gCoreEntries[thisCore].fThreadCount, 1); + atomic_add(&gCoreEntries[thisCore].fStarvationCounter, 1); } if (sharedThread->scheduler_data->went_sleep_count == 0) { @@ -948,6 +959,7 @@ choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) sharedThread->scheduler_data); } + atomic_add(&gCoreEntries[thisCore].fThreadCount, -1); return sharedThread; } @@ -1250,7 +1262,7 @@ reschedule(void) oldThread->cpu->preempted = false; if (!thread_is_idle_thread(nextThread)) { - bigtime_t quantum = compute_quantum(oldThread); + bigtime_t quantum = compute_quantum(nextThread); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); } else { @@ -1430,6 +1442,7 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) // get rid of threads thread_map(unassign_thread, &core->fCoreID); + core->fThreadCount = 0; while (gRunQueues[core->fCoreID].PeekMaximum() != NULL) { Thread* thread = gRunQueues[core->fCoreID].PeekMaximum(); gRunQueues[core->fCoreID].Remove(thread); diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index e92b3a6763..4c63c9fe9b 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -29,7 +29,8 @@ namespace Scheduler { -const bigtime_t kThreadQuantum = 1000; +const bigtime_t kThreadQuantum = 2000; +const bigtime_t kMaximumLatency = 600000; const bigtime_t kCacheExpire = 100000; @@ -78,6 +79,8 @@ struct CoreEntry : public MinMaxHeapLinkImpl, spinlock fCPULock; spinlock fQueueLock; + int32 fStarvationCounter; + int32 fThreadCount; DoublyLinkedList fThreadList; From e736a456ba4d12621654a3f95c394d11fcaac243 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 28 Nov 2013 23:28:33 +0100 Subject: [PATCH 160/273] kernel: Forbid implicit casts between spinlock and int32 --- headers/os/drivers/KernelExport.h | 14 +++++--- .../kernel/bus_managers/firewire/timer.cpp | 2 +- .../drivers/audio/ac97/sis7018/Device.cpp | 3 +- .../drivers/audio/ac97/sis7018/Device.h | 2 +- src/add-ons/kernel/drivers/audio/echo/util.c | 2 +- .../kernel/drivers/audio/ice1712/util.c | 2 +- .../drivers/disk/floppy/pc_floppy/floppy.c | 2 +- .../kernel/drivers/network/sis19x/DataRing.h | 4 +-- .../kernel/drivers/network/sis19x/Device.cpp | 3 +- .../kernel/drivers/network/sis19x/Device.h | 2 +- src/system/kernel/smp.cpp | 32 ++++++++++--------- 11 files changed, 38 insertions(+), 30 deletions(-) diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index de40d0c76d..4f7e8e2969 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -31,15 +31,19 @@ typedef ulong cpu_status; (spinlock)->count_low = 0; \ (spinlock)->count_high = 0; \ } while (false) -# define B_SPINLOCK_IS_LOCKED(spinlock) ((spinlock)->lock > 0) #else - typedef int32 spinlock; + typedef struct { + int32 lock; + } spinlock; -# define B_SPINLOCK_INITIALIZER 0 -# define B_INITIALIZE_SPINLOCK(lock) do { *(lock) = 0; } while (false) -# define B_SPINLOCK_IS_LOCKED(lock) (*(lock) > 0) +# define B_SPINLOCK_INITIALIZER { 0 } +# define B_INITIALIZE_SPINLOCK(spinlock) do { \ + (spinlock)->lock = 0; \ + } while (false) #endif +#define B_SPINLOCK_IS_LOCKED(spinlock) (atomic_get(&(spinlock)->lock) > 0) + typedef struct { int32 lock; } rw_spinlock; diff --git a/src/add-ons/kernel/bus_managers/firewire/timer.cpp b/src/add-ons/kernel/bus_managers/firewire/timer.cpp index 7299d9037b..4df8780784 100644 --- a/src/add-ons/kernel/bus_managers/firewire/timer.cpp +++ b/src/add-ons/kernel/bus_managers/firewire/timer.cpp @@ -188,7 +188,7 @@ initialize_timer(void) { sTimerCount = 0; sTimerNextId = 1; - sTimerSpinlock = 0; + B_INITIALIZE_SPINLOCK(&sTimerSpinlock); sTimerThread = spawn_kernel_thread(timer_thread, "firewire timer", 80, 0); sTimerSem = create_sem(0, "firewire timer"); diff --git a/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.cpp b/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.cpp index e432ec88b2..80c1ab4fad 100644 --- a/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.cpp +++ b/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.cpp @@ -22,13 +22,14 @@ Device::Device(Device::Info &DeviceInfo, pci_info &PCIInfo) fPCIInfo(PCIInfo), fInfo(DeviceInfo), fIOBase(0), - fHWSpinlock(0), fInterruptsNest(0), fBuffersReadySem(-1), fMixer(this), fPlaybackStream(this, false), fRecordStream(this, true) { + B_INITIALIZE_SPINLOCK(&fHWSpinlock); + fStatus = _ReserveDeviceOnBus(true); if (fStatus != B_OK) return; // InitCheck will handle the rest diff --git a/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.h b/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.h index 399161a33d..974d781d0a 100644 --- a/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.h +++ b/src/add-ons/kernel/drivers/audio/ac97/sis7018/Device.h @@ -88,7 +88,7 @@ private: pci_info fPCIInfo; Info& fInfo; int fIOBase; - int32 fHWSpinlock; + spinlock fHWSpinlock; int32 fInterruptsNest; sem_id fBuffersReadySem; diff --git a/src/add-ons/kernel/drivers/audio/echo/util.c b/src/add-ons/kernel/drivers/audio/echo/util.c index 950ad76135..a35107607f 100644 --- a/src/add-ons/kernel/drivers/audio/echo/util.c +++ b/src/add-ons/kernel/drivers/audio/echo/util.c @@ -34,7 +34,7 @@ #include "debug.h" #include "util.h" -spinlock slock = 0; +spinlock slock = B_SPINLOCK_INITIALIZER; uint32 round_to_pagesize(uint32 size); diff --git a/src/add-ons/kernel/drivers/audio/ice1712/util.c b/src/add-ons/kernel/drivers/audio/ice1712/util.c index c036e73fec..2feea676aa 100644 --- a/src/add-ons/kernel/drivers/audio/ice1712/util.c +++ b/src/add-ons/kernel/drivers/audio/ice1712/util.c @@ -16,7 +16,7 @@ #include "debug.h" #include "util.h" -spinlock slock = 0; +spinlock slock = B_SPINLOCK_INITIALIZER; uint32 round_to_pagesize(uint32 size); diff --git a/src/add-ons/kernel/drivers/disk/floppy/pc_floppy/floppy.c b/src/add-ons/kernel/drivers/disk/floppy/pc_floppy/floppy.c index e2f4ed7263..b78cda248d 100644 --- a/src/add-ons/kernel/drivers/disk/floppy/pc_floppy/floppy.c +++ b/src/add-ons/kernel/drivers/disk/floppy/pc_floppy/floppy.c @@ -222,7 +222,7 @@ init_driver(void) B_FULL_LOCK, B_READ_AREA|B_WRITE_AREA); if (master->buffer_area < B_OK) goto config_error2; - master->slock = 0; + B_INITIALIZE_SPINLOCK(&master->slock); master->isa = isa; if (install_io_interrupt_handler(master->irq, flo_intr, (void *)master, 0) < B_OK) goto config_error2; diff --git a/src/add-ons/kernel/drivers/network/sis19x/DataRing.h b/src/add-ons/kernel/drivers/network/sis19x/DataRing.h index af4ebfb625..a3e74aa7e0 100644 --- a/src/add-ons/kernel/drivers/network/sis19x/DataRing.h +++ b/src/add-ons/kernel/drivers/network/sis19x/DataRing.h @@ -43,7 +43,7 @@ private: bool fIsTx; status_t fStatus; area_id fArea; - int32 fSpinlock; + spinlock fSpinlock; sem_id fSemaphore; uint32 fHead; uint32 fTail; @@ -61,12 +61,12 @@ DataRing<__type, __count>::DataRing(Device* device, bool isTx) fIsTx(isTx), fStatus(B_NO_INIT), fArea(-1), - fSpinlock(0), fSemaphore(0), fHead(0), fTail(0), fDescriptors(NULL) { + B_INITIALIZE_SPINLOCK(&fSpinlock); memset(fBuffers, 0, sizeof(fBuffers)); } diff --git a/src/add-ons/kernel/drivers/network/sis19x/Device.cpp b/src/add-ons/kernel/drivers/network/sis19x/Device.cpp index feb94f512a..fb396eadbf 100644 --- a/src/add-ons/kernel/drivers/network/sis19x/Device.cpp +++ b/src/add-ons/kernel/drivers/network/sis19x/Device.cpp @@ -23,7 +23,6 @@ Device::Device(Device::Info &DeviceInfo, pci_info &PCIInfo) fPCIInfo(PCIInfo), fInfo(DeviceInfo), fIOBase(0), - fHWSpinlock(0), fInterruptsNest(0), fFrameSize(MaxFrameSize), fMII(this), @@ -36,6 +35,8 @@ Device::Device(Device::Info &DeviceInfo, pci_info &PCIInfo) { memset((struct timer*)this, 0, sizeof(struct timer)); + B_INITIALIZE_SPINLOCK(&fHWSpinlock); + uint32 cmdRegister = gPCIModule->read_pci_config(PCIInfo.bus, PCIInfo.device, PCIInfo.function, PCI_command, 2); TRACE_ALWAYS("cmdRegister:%#010x\n", cmdRegister); diff --git a/src/add-ons/kernel/drivers/network/sis19x/Device.h b/src/add-ons/kernel/drivers/network/sis19x/Device.h index 30aa6c5eb2..3d73b1326b 100644 --- a/src/add-ons/kernel/drivers/network/sis19x/Device.h +++ b/src/add-ons/kernel/drivers/network/sis19x/Device.h @@ -97,7 +97,7 @@ static int32 _TimerHandler(struct timer* timer); pci_info fPCIInfo; Info& fInfo; int fIOBase; - int32 fHWSpinlock; + spinlock fHWSpinlock; int32 fInterruptsNest; // interface and device infos diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 0477b85129..cd25254668 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. * Distributed under the terms of the MIT License. @@ -26,6 +27,7 @@ #include #include #include +#include #if DEBUG_SPINLOCK_LATENCIES # include #endif @@ -350,7 +352,7 @@ acquire_spinlock(spinlock* lock) #else while (1) { uint32 count = 0; - while (atomic_get(lock) != 0) { + while (atomic_get(&lock->lock) != 0) { if (++count == SPINLOCK_DEADLOCK_COUNT) { panic("acquire_spinlock(): Failed to acquire spinlock %p " "for a long time!", lock); @@ -358,9 +360,9 @@ acquire_spinlock(spinlock* lock) } process_all_pending_ici(currentCPU); - cpu_wait(lock, 0); + cpu_wait(&lock->lock, 0); } - if (atomic_get_and_set((int32*)lock, 1) == 0) + if (atomic_get_and_set(&lock->lock, 1) == 0) break; } @@ -371,7 +373,7 @@ acquire_spinlock(spinlock* lock) } else { #if DEBUG_SPINLOCKS int32 oldValue; - oldValue = atomic_get_and_set((int32*)lock, 1); + oldValue = atomic_get_and_set(&lock->lock, 1); if (oldValue != 0) { panic("acquire_spinlock: attempt to acquire lock %p twice on " "non-SMP system (last caller: %p, value %" B_PRId32 ")", lock, @@ -404,23 +406,23 @@ acquire_spinlock_nocheck(spinlock *lock) #else while (1) { uint32 count = 0; - while (atomic_get(lock) != 0) { + while (atomic_get(&lock->lock) != 0) { if (++count == SPINLOCK_DEADLOCK_COUNT_NO_CHECK) { panic("acquire_spinlock(): Failed to acquire spinlock %p " "for a long time!", lock); count = 0; } - cpu_wait(lock, 0); + cpu_wait(&lock->lock, 0); } - if (atomic_get_and_set((int32*)lock, 1) == 0) + if (atomic_get_and_set(&lock->lock, 1) == 0) break; } #endif } else { #if DEBUG_SPINLOCKS - if (atomic_get_and_set((int32*)lock, 1) != 0) { + if (atomic_get_and_set(&lock->lock, 1) != 0) { panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice " "on non-SMP system\n", lock); } @@ -447,7 +449,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) #else while (1) { uint32 count = 0; - while (atomic_get(lock) != 0) { + while (atomic_get(&lock->lock) != 0) { if (++count == SPINLOCK_DEADLOCK_COUNT) { panic("acquire_spinlock_cpu(): Failed to acquire spinlock " "%p for a long time!", lock); @@ -455,9 +457,9 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) } process_all_pending_ici(currentCPU); - cpu_wait(lock, 0); + cpu_wait(&lock->lock, 0); } - if (atomic_get_and_set((int32*)lock, 1) == 0) + if (atomic_get_and_set(&lock->lock, 1) == 0) break; } @@ -468,7 +470,7 @@ acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) } else { #if DEBUG_SPINLOCKS int32 oldValue; - oldValue = atomic_get_and_set((int32*)lock, 1); + oldValue = atomic_get_and_set(&lock->lock, 1); if (oldValue != 0) { panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " "non-SMP system (last caller: %p, value %" B_PRId32 ")", lock, @@ -506,10 +508,10 @@ release_spinlock(spinlock *lock) } } #elif DEBUG_SPINLOCKS - if (atomic_get_and_set((int32*)lock, 0) != 1) + if (atomic_get_and_set(&lock->lock, 0) != 1) panic("release_spinlock: lock %p was already released\n", lock); #else - atomic_set((int32*)lock, 0); + atomic_set(&lock->lock, 0); #endif } else { #if DEBUG_SPINLOCKS @@ -517,7 +519,7 @@ release_spinlock(spinlock *lock) panic("release_spinlock: attempt to release lock %p with " "interrupts enabled\n", lock); } - if (atomic_get_and_set((int32*)lock, 0) != 1) + if (atomic_get_and_set(&lock->lock, 0) != 1) panic("release_spinlock: lock %p was already released\n", lock); #endif #if DEBUG_SPINLOCK_LATENCIES From 3514fd77f702359e815201419aebded48f032ad8 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 03:27:00 +0100 Subject: [PATCH 161/273] kernel: Reduce lock contention when processing ICIs --- headers/private/kernel/arch/x86/arch_cpu.h | 2 +- headers/private/kernel/cpu.h | 2 + src/system/kernel/smp.cpp | 193 +++++++++++---------- 3 files changed, 105 insertions(+), 92 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index f4ccaf487c..502b33b6c2 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -497,7 +497,7 @@ arch_cpu_idle(void) static inline void arch_cpu_pause(void) { - asm volatile("pause"); + asm volatile("pause" : : : "memory"); } diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 3337d387d4..7b4d51f81a 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -60,6 +60,8 @@ typedef struct cpu_ent { bigtime_t last_kernel_time; bigtime_t last_user_time; + int32 ici_counter; + // used in the kernel debugger addr_t fault_handler; addr_t fault_handler_stack_pointer; diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index cd25254668..9788b2b343 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -37,9 +37,9 @@ //#define TRACE_SMP #ifdef TRACE_SMP -# define TRACE(x) dprintf x +# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) #else -# define TRACE(x) ; +# define TRACE(...) (void)0 #endif @@ -79,7 +79,7 @@ struct smp_msg { void *data_ptr; uint32 flags; int32 ref_count; - volatile bool done; + int32 done; uint32 proc_bitmap; }; @@ -95,14 +95,14 @@ static void (*sEarlyCPUCallFunction)(void*, int); void* sEarlyCPUCallCookie; static struct smp_msg* sFreeMessages = NULL; -static volatile int sFreeMessageCount = 0; +static int32 sFreeMessageCount = 0; static spinlock sFreeMessageSpinlock = B_SPINLOCK_INITIALIZER; static struct smp_msg* sCPUMessages[SMP_MAX_CPUS] = { NULL, }; -static spinlock sCPUMessageSpinlock[SMP_MAX_CPUS]; static struct smp_msg* sBroadcastMessages = NULL; static spinlock sBroadcastMessageSpinlock = B_SPINLOCK_INITIALIZER; +static int32 sBroadcastMessageCounter; static bool sICIEnabled = false; static int32 sNumCPUs = 1; @@ -242,7 +242,7 @@ dump_ici_messages(int argc, char** argv) smp_msg* message = sBroadcastMessages; while (message != NULL) { count++; - if (message->done) + if (message->done == 1) doneCount++; if (message->ref_count <= 0) unreferencedCount++; @@ -293,7 +293,7 @@ dump_ici_message(int argc, char** argv) kprintf(" data_ptr: %p\n", message->data_ptr); kprintf(" flags: %" B_PRIx32 "\n", message->flags); kprintf(" ref_count: %" B_PRIx32 "\n", message->ref_count); - kprintf(" done: %s\n", message->done ? "true" : "false"); + kprintf(" done: %s\n", message->done == 1 ? "true" : "false"); kprintf(" proc_bitmap: %" B_PRIx32 "\n", message->proc_bitmap); return 0; @@ -714,15 +714,12 @@ find_free_message(struct smp_msg** msg) { cpu_status state; - TRACE(("find_free_message: entry\n")); + TRACE("find_free_message: entry\n"); retry: - while (sFreeMessageCount <= 0) { - state = disable_interrupts(); - process_all_pending_ici(smp_get_current_cpu()); - restore_interrupts(state); + while (atomic_get(&sFreeMessageCount) <= 0) cpu_pause(); - } + state = disable_interrupts(); acquire_spinlock(&sFreeMessageSpinlock); @@ -740,7 +737,7 @@ retry: release_spinlock(&sFreeMessageSpinlock); - TRACE(("find_free_message: returning msg %p\n", *msg)); + TRACE("find_free_message: returning msg %p\n", *msg); return state; } @@ -753,10 +750,10 @@ static void find_free_message_interrupts_disabled(int32 currentCPU, struct smp_msg** _message) { - TRACE(("find_free_message_interrupts_disabled: entry\n")); + TRACE("find_free_message_interrupts_disabled: entry\n"); acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); - while (sFreeMessageCount <= 0) { + while (atomic_get(&sFreeMessageCount) <= 0) { release_spinlock(&sFreeMessageSpinlock); process_all_pending_ici(currentCPU); cpu_pause(); @@ -769,15 +766,15 @@ find_free_message_interrupts_disabled(int32 currentCPU, release_spinlock(&sFreeMessageSpinlock); - TRACE(("find_free_message_interrupts_disabled: returning msg %p\n", - *_message)); + TRACE("find_free_message_interrupts_disabled: returning msg %p\n", + *_message); } static void return_free_message(struct smp_msg* msg) { - TRACE(("return_free_message: returning msg %p\n", msg)); + TRACE("return_free_message: returning msg %p\n", msg); acquire_spinlock_nocheck(&sFreeMessageSpinlock); msg->next = sFreeMessages; @@ -793,18 +790,21 @@ check_for_message(int currentCPU, mailbox_source& sourceMailbox) if (!sICIEnabled) return NULL; - acquire_spinlock_nocheck(&sCPUMessageSpinlock[currentCPU]); - - struct smp_msg* msg = sCPUMessages[currentCPU]; + struct smp_msg* msg = atomic_pointer_get(&sCPUMessages[currentCPU]); if (msg != NULL) { - sCPUMessages[currentCPU] = msg->next; - release_spinlock(&sCPUMessageSpinlock[currentCPU]); - TRACE((" cpu %d: found msg %p in cpu mailbox\n", currentCPU, msg)); - sourceMailbox = MAILBOX_LOCAL; - } else { - // try getting one from the broadcast mailbox + do { + cpu_pause(); + msg = atomic_pointer_get(&sCPUMessages[currentCPU]); + ASSERT(msg != NULL); + } while (atomic_pointer_test_and_set(&sCPUMessages[currentCPU], + msg->next, msg) != msg); - release_spinlock(&sCPUMessageSpinlock[currentCPU]); + TRACE(" cpu %d: found msg %p in cpu mailbox\n", currentCPU, msg); + sourceMailbox = MAILBOX_LOCAL; + } else if (atomic_get(&get_cpu_struct()->ici_counter) + != atomic_get(&sBroadcastMessageCounter)) { + + // try getting one from the broadcast mailbox acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); msg = sBroadcastMessages; @@ -817,13 +817,17 @@ check_for_message(int currentCPU, mailbox_source& sourceMailbox) // mark it so we wont try to process this one again msg->proc_bitmap = SET_BIT(msg->proc_bitmap, currentCPU); + atomic_add(&gCPU[currentCPU].ici_counter, 1); + sourceMailbox = MAILBOX_BCAST; break; } release_spinlock(&sBroadcastMessageSpinlock); - TRACE((" cpu %d: found msg %p in broadcast mailbox\n", currentCPU, - msg)); + if (msg != NULL) { + TRACE(" cpu %d: found msg %p in broadcast mailbox\n", currentCPU, + msg); + } } return msg; } @@ -838,27 +842,18 @@ finish_message_processing(int currentCPU, struct smp_msg* msg, // we were the last one to decrement the ref_count // it's our job to remove it from the list & possibly clean it up - struct smp_msg** mbox; - spinlock* spinlock; - // clean up the message from one of the mailboxes - if (sourceMailbox == MAILBOX_BCAST) { - mbox = &sBroadcastMessages; - spinlock = &sBroadcastMessageSpinlock; - } else { - mbox = &sCPUMessages[currentCPU]; - spinlock = &sCPUMessageSpinlock[currentCPU]; - } + // clean up the message + if (sourceMailbox == MAILBOX_BCAST) + acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); - acquire_spinlock_nocheck(spinlock); - - TRACE(("cleaning up message %p\n", msg)); + TRACE("cleaning up message %p\n", msg); if (sourceMailbox != MAILBOX_BCAST) { // local mailbox -- the message has already been removed in // check_for_message() - } else if (msg == *mbox) { - *mbox = msg->next; + } else if (msg == sBroadcastMessages) { + sBroadcastMessages = msg->next; } else { // we need to walk to find the message in the list. // we can't use any data found when previously walking through @@ -867,7 +862,7 @@ finish_message_processing(int currentCPU, struct smp_msg* msg, struct smp_msg* last = NULL; struct smp_msg* msg1; - msg1 = *mbox; + msg1 = sBroadcastMessages; while (msg1 != NULL && msg1 != msg) { last = msg1; msg1 = msg1->next; @@ -880,13 +875,14 @@ finish_message_processing(int currentCPU, struct smp_msg* msg, panic("last == NULL or msg != msg1"); } - release_spinlock(spinlock); + if (sourceMailbox == MAILBOX_BCAST) + release_spinlock(&sBroadcastMessageSpinlock); if ((msg->flags & SMP_MSG_FLAG_FREE_ARG) != 0 && msg->data_ptr != NULL) free(msg->data_ptr); if ((msg->flags & SMP_MSG_FLAG_SYNC) != 0) { - msg->done = true; + atomic_set(&msg->done, 1); // the caller cpu should now free the message } else { // in the !SYNC case, we get to free the message @@ -903,7 +899,7 @@ process_pending_ici(int32 currentCPU) if (msg == NULL) return B_ENTRY_NOT_FOUND; - TRACE((" cpu %ld message = %ld\n", currentCPU, msg->message)); + TRACE(" cpu %ld message = %ld\n", currentCPU, msg->message); bool haltCPU = false; @@ -1029,11 +1025,11 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie) int smp_intercpu_int_handler(int32 cpu) { - TRACE(("smp_intercpu_int_handler: entry on cpu %ld\n", cpu)); + TRACE("smp_intercpu_int_handler: entry on cpu %ld\n", cpu); process_all_pending_ici(cpu); - TRACE(("smp_intercpu_int_handler: done\n")); + TRACE("smp_intercpu_int_handler: done on cpu %ld\n", cpu); return B_HANDLED_INTERRUPT; } @@ -1045,9 +1041,9 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, { struct smp_msg *msg; - TRACE(("smp_send_ici: target 0x%lx, mess 0x%lx, data 0x%lx, data2 0x%lx, " + TRACE("smp_send_ici: target 0x%lx, mess 0x%lx, data 0x%lx, data2 0x%lx, " "data3 0x%lx, ptr %p, flags 0x%lx\n", targetCPU, message, data, data2, - data3, dataPointer, flags)); + data3, dataPointer, flags); if (sICIEnabled) { int state; @@ -1071,13 +1067,16 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, msg->data_ptr = dataPointer; msg->ref_count = 1; msg->flags = flags; - msg->done = false; + msg->done = 0; // stick it in the appropriate cpu's mailbox - acquire_spinlock_nocheck(&sCPUMessageSpinlock[targetCPU]); - msg->next = sCPUMessages[targetCPU]; - sCPUMessages[targetCPU] = msg; - release_spinlock(&sCPUMessageSpinlock[targetCPU]); + struct smp_msg* next; + do { + cpu_pause(); + next = atomic_pointer_get(&sCPUMessages[targetCPU]); + msg->next = next; + } while (atomic_pointer_test_and_set(&sCPUMessages[targetCPU], msg, + next) != next); arch_smp_send_ici(targetCPU); @@ -1085,9 +1084,9 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, // wait for the other cpu to finish processing it // the interrupt handler will ref count it to <0 // if the message is sync after it has removed it from the mailbox - while (msg->done == false) { + while (atomic_get(&msg->done) == 0) { process_all_pending_ici(currentCPU); - cpu_pause(); + cpu_wait(&msg->done, 1); } // for SYNC messages, it's our responsibility to put it // back into the free list @@ -1133,7 +1132,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, msg->ref_count = targetCPUs; msg->flags = flags; msg->proc_bitmap = ~cpuMask; - msg->done = false; + msg->done = 0; // stick it in the broadcast mailbox acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); @@ -1141,6 +1140,12 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, sBroadcastMessages = msg; release_spinlock(&sBroadcastMessageSpinlock); + atomic_add(&sBroadcastMessageCounter, 1); + for (int32 i = 0; i < sNumCPUs; i++) { + if ((cpuMask & (cpu_mask_t)1 << i) == 0) + atomic_add(&gCPU[i].ici_counter, 1); + } + arch_smp_send_broadcast_ici(); // TODO: Introduce a call that only bothers the target CPUs! @@ -1148,9 +1153,9 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, // wait for the other cpus to finish processing it // the interrupt handler will ref count it to <0 // if the message is sync after it has removed it from the mailbox - while (msg->done == false) { + while (atomic_get(&msg->done) == 0) { process_all_pending_ici(currentCPU); - cpu_pause(); + cpu_wait(&msg->done, 1); } // for SYNC messages, it's our responsibility to put it @@ -1168,9 +1173,9 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, { struct smp_msg *msg; - TRACE(("smp_send_broadcast_ici: cpu %ld mess 0x%lx, data 0x%lx, data2 " + TRACE("smp_send_broadcast_ici: cpu %ld mess 0x%lx, data 0x%lx, data2 " "0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", smp_get_current_cpu(), - message, data, data2, data3, dataPointer, flags)); + message, data, data2, data3, dataPointer, flags); if (sICIEnabled) { int state; @@ -1189,10 +1194,10 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, msg->ref_count = sNumCPUs - 1; msg->flags = flags; msg->proc_bitmap = SET_BIT(0, currentCPU); - msg->done = false; + msg->done = 0; - TRACE(("smp_send_broadcast_ici%d: inserting msg %p into broadcast " - "mbox\n", currentCPU, msg)); + TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast " + "mbox\n", currentCPU, msg); // stick it in the appropriate cpu's mailbox acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); @@ -1200,22 +1205,25 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, sBroadcastMessages = msg; release_spinlock(&sBroadcastMessageSpinlock); + atomic_add(&sBroadcastMessageCounter, 1); + atomic_add(&gCPU[currentCPU].ici_counter, 1); + arch_smp_send_broadcast_ici(); - TRACE(("smp_send_broadcast_ici: sent interrupt\n")); + TRACE("smp_send_broadcast_ici: sent interrupt\n"); if ((flags & SMP_MSG_FLAG_SYNC) != 0) { // wait for the other cpus to finish processing it // the interrupt handler will ref count it to <0 // if the message is sync after it has removed it from the mailbox - TRACE(("smp_send_broadcast_ici: waiting for ack\n")); + TRACE("smp_send_broadcast_ici: waiting for ack\n"); - while (msg->done == false) { + while (atomic_get(&msg->done) == 0) { process_all_pending_ici(currentCPU); - cpu_pause(); + cpu_wait(&msg->done, 1); } - TRACE(("smp_send_broadcast_ici: returning message to free list\n")); + TRACE("smp_send_broadcast_ici: returning message to free list\n"); // for SYNC messages, it's our responsibility to put it // back into the free list @@ -1225,7 +1233,7 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, restore_interrupts(state); } - TRACE(("smp_send_broadcast_ici: done\n")); + TRACE("smp_send_broadcast_ici: done\n"); } @@ -1236,9 +1244,9 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, if (!sICIEnabled) return; - TRACE(("smp_send_broadcast_ici_interrupts_disabled: cpu %ld mess 0x%lx, " + TRACE("smp_send_broadcast_ici_interrupts_disabled: cpu %ld mess 0x%lx, " "data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", - currentCPU, message, data, data2, data3, dataPointer, flags)); + currentCPU, message, data, data2, data3, dataPointer, flags); struct smp_msg *msg; find_free_message_interrupts_disabled(currentCPU, &msg); @@ -1251,10 +1259,10 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, msg->ref_count = sNumCPUs - 1; msg->flags = flags; msg->proc_bitmap = SET_BIT(0, currentCPU); - msg->done = false; + msg->done = 0; - TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " - "into broadcast mbox\n", currentCPU, msg)); + TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " + "into broadcast mbox\n", currentCPU, msg); // stick it in the appropriate cpu's mailbox acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); @@ -1262,32 +1270,35 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, sBroadcastMessages = msg; release_spinlock(&sBroadcastMessageSpinlock); + atomic_add(&sBroadcastMessageCounter, 1); + atomic_add(&gCPU[currentCPU].ici_counter, 1); + arch_smp_send_broadcast_ici(); - TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: sent interrupt\n", - currentCPU)); + TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: sent interrupt\n", + currentCPU); if ((flags & SMP_MSG_FLAG_SYNC) != 0) { // wait for the other cpus to finish processing it // the interrupt handler will ref count it to <0 // if the message is sync after it has removed it from the mailbox - TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: waiting for " - "ack\n", currentCPU)); + TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: waiting for " + "ack\n", currentCPU); - while (msg->done == false) { + while (atomic_get(&msg->done) == 0) { process_all_pending_ici(currentCPU); - cpu_pause(); + cpu_wait(&msg->done, 1); } - TRACE(("smp_send_broadcast_ici_interrupts_disabled %ld: returning " - "message to free list\n", currentCPU)); + TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: returning " + "message to free list\n", currentCPU); // for SYNC messages, it's our responsibility to put it // back into the free list return_free_message(msg); } - TRACE(("smp_send_broadcast_ici_interrupts_disabled: done\n")); + TRACE("smp_send_broadcast_ici_interrupts_disabled: done\n"); } @@ -1355,7 +1366,7 @@ smp_cpu_rendezvous(uint32* var, int current_cpu) status_t smp_init(kernel_args* args) { - TRACE(("smp_init: entry\n")); + TRACE("smp_init: entry\n"); #if DEBUG_SPINLOCK_LATENCIES sEnableLatencyCheck @@ -1394,7 +1405,7 @@ smp_init(kernel_args* args) } sNumCPUs = args->num_cpus; } - TRACE(("smp_init: calling arch_smp_init\n")); + TRACE("smp_init: calling arch_smp_init\n"); return arch_smp_init(args); } From 1ee1d0cbab735cd0849fefaa1ee62ae8fe83d7a5 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 03:47:38 +0100 Subject: [PATCH 162/273] kernel, drivers: Fix gcc2 build --- src/add-ons/kernel/bus_managers/acpi/ACPICAHaiku.cpp | 2 +- src/system/kernel/locks/lock.cpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/add-ons/kernel/bus_managers/acpi/ACPICAHaiku.cpp b/src/add-ons/kernel/bus_managers/acpi/ACPICAHaiku.cpp index d001e0e2be..68af7542ac 100644 --- a/src/add-ons/kernel/bus_managers/acpi/ACPICAHaiku.cpp +++ b/src/add-ons/kernel/bus_managers/acpi/ACPICAHaiku.cpp @@ -676,7 +676,7 @@ AcpiOsCreateLock(ACPI_SPINLOCK *outHandle) if (*outHandle == NULL) return AE_NO_MEMORY; - **outHandle = B_SPINLOCK_INITIALIZER; + B_INITIALIZE_SPINLOCK(*outHandle); return AE_OK; } diff --git a/src/system/kernel/locks/lock.cpp b/src/system/kernel/locks/lock.cpp index 3240549fbd..facf6268e5 100644 --- a/src/system/kernel/locks/lock.cpp +++ b/src/system/kernel/locks/lock.cpp @@ -231,7 +231,7 @@ rw_lock_init(rw_lock* lock, const char* name) { lock->name = name; lock->waiters = NULL; - lock->lock = B_SPINLOCK_INITIALIZER; + B_INITIALIZE_SPINLOCK(&lock->lock); lock->holder = -1; lock->count = 0; lock->owner_count = 0; @@ -249,7 +249,7 @@ rw_lock_init_etc(rw_lock* lock, const char* name, uint32 flags) { lock->name = (flags & RW_LOCK_FLAG_CLONE_NAME) != 0 ? strdup(name) : name; lock->waiters = NULL; - lock->lock = B_SPINLOCK_INITIALIZER; + B_INITIALIZE_SPINLOCK(&lock->lock); lock->holder = -1; lock->count = 0; lock->owner_count = 0; @@ -574,7 +574,7 @@ mutex_init(mutex* lock, const char *name) { lock->name = name; lock->waiters = NULL; - lock->lock = B_SPINLOCK_INITIALIZER; + B_INITIALIZE_SPINLOCK(&lock->lock); #if KDEBUG lock->holder = -1; #else @@ -593,7 +593,7 @@ mutex_init_etc(mutex* lock, const char *name, uint32 flags) { lock->name = (flags & MUTEX_FLAG_CLONE_NAME) != 0 ? strdup(name) : name; lock->waiters = NULL; - lock->lock = B_SPINLOCK_INITIALIZER; + B_INITIALIZE_SPINLOCK(&lock->lock); #if KDEBUG lock->holder = -1; #else From 1987bf19b57e2789bbf4cc736f81cfa803a62af2 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 04:35:50 +0100 Subject: [PATCH 163/273] scheduler: Update CPU disabling code to match recent changes --- src/system/kernel/scheduler/scheduler.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ee659d1732..65e3549852 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1422,7 +1422,7 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) ASSERT(!enabled); int32 load = CoreLoadHeap::GetKey(core); - if (load > kHighLoad) { + if (core->fHighLoad) { gCoreHighLoadHeap->ModifyKey(core, -1); ASSERT(gCoreHighLoadHeap->PeekMinimum() == core); gCoreHighLoadHeap->RemoveMinimum(); @@ -1448,6 +1448,12 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) gRunQueues[core->fCoreID].Remove(thread); thread->scheduler_data->enqueued = false; + if (thread->scheduler_data->went_sleep_count == 0) { + gCoreEntries[core->fCoreID].fThreadList.Remove( + thread->scheduler_data); + thread->scheduler_data->went_sleep_count = -1;; + } + ASSERT(thread->scheduler_data->previous_core == -1); enqueue(thread, false); } @@ -1457,6 +1463,7 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) gCPUEntries[cpu].fLoad = 0; core->fLoad = 0; + core->fHighLoad = false; gCoreLoadHeap->Insert(core, 0); package->fCoreCount++; @@ -1478,7 +1485,11 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) == &gCPUEntries[cpu]); gCPUPriorityHeaps[core->fCoreID].RemoveMaximum(); + ASSERT(gCPUEntries[cpu].fLoad >= 0 + && gCPUEntries[cpu].fLoad <= kMaxLoad); + core->fLoad -= gCPUEntries[cpu].fLoad; + ASSERT(core->fLoad >= 0); } if (!enabled) { From 7f8cf14b711fae638e2c089afb9513461513100f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 04:51:38 +0100 Subject: [PATCH 164/273] scheduler, kernel/util: Fix style issues --- headers/private/kernel/util/MinMaxHeap.h | 4 ++-- src/system/kernel/scheduler/RunQueue.h | 2 +- src/system/kernel/scheduler/scheduler.cpp | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 6e7caaea29..cc4ab0a7a0 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -198,7 +198,7 @@ MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() ASSERT(fMaxLastElement == 1); return fMaxElements[0]; } - + return NULL; } @@ -213,7 +213,7 @@ MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() ASSERT(fMinLastElement == 1); return fMinElements[0]; } - + return NULL; } diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index e33f1ab1bd..a38fb32f5d 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -156,7 +156,7 @@ RUN_QUEUE_CLASS_NAME::ConstIterator::Next() fNext = link->fNext; if (fNext == NULL) _FindNextPriority(); - + return current; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 65e3549852..9ec5d217d2 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -830,7 +830,7 @@ reschedule_event(timer* /* unused */) } -static inline bool +static inline bool quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) { scheduler_thread_data* schedulerThreadData = thread->scheduler_data; @@ -902,7 +902,7 @@ compute_quantum(Thread* thread) == gCPUToCore[smp_get_current_cpu()]); CoreEntry* core = &gCoreEntries[schedulerThreadData->previous_core]; int32 threadCount = (core->fThreadCount + 1) / core->fCPUCount; - if (threadCount > 1) { + if (threadCount > 1) { quantum = max_c(min_c(kMaximumLatency / threadCount, quantum), kThreadQuantum / 3); } @@ -1252,7 +1252,7 @@ reschedule(void) nextThread->scheduler_data->last_interrupt_time = gCPU[thisCPU].interrupt_time; - if (!thread_is_idle_thread(nextThread)) + if (!thread_is_idle_thread(nextThread)) update_cpu_performance(nextThread, thisCore); if (nextThread != oldThread || oldThread->cpu->preempted) { @@ -1311,7 +1311,7 @@ scheduler_on_thread_init(Thread* thread) thread->scheduler_data->Init(); if (thread_is_idle_thread(thread)) { - static int32 gIdleThreadsID; + static int32 sIdleThreadsID; int32 cpu = atomic_add(&gIdleThreadsID, 1); thread->previous_cpu = &gCPU[cpu]; @@ -1344,7 +1344,7 @@ scheduler_start(void) static inline void acquire_big_scheduler_lock(void) { - for (int32_t i = 0; i < smp_get_num_cpus(); i++) + for (int32 i = 0; i < smp_get_num_cpus(); i++) acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); } @@ -1352,7 +1352,7 @@ acquire_big_scheduler_lock(void) static inline void release_big_scheduler_lock(void) { - for (int32_t i = 0; i < smp_get_num_cpus(); i++) + for (int32 i = 0; i < smp_get_num_cpus(); i++) release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); } @@ -1471,7 +1471,7 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) package->fIdleCores.Add(core); if (package->fCoreCount == 1) - gIdlePackageList->Add(package); + gIdlePackageList->Add(package); } if (enabled) { From 072189970913d3cce35c34b4307b5e58b0f241b5 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 05:11:44 +0100 Subject: [PATCH 165/273] scheduler: Allow mode to specify time slice length --- src/system/kernel/scheduler/low_latency.cpp | 9 +++++ src/system/kernel/scheduler/power_saving.cpp | 8 ++++ src/system/kernel/scheduler/scheduler.cpp | 37 +++++++++++-------- .../kernel/scheduler/scheduler_common.h | 5 --- src/system/kernel/scheduler/scheduler_modes.h | 6 +++ 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 3e4365768b..6adfe4e6fb 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -13,6 +13,9 @@ using namespace Scheduler; +const bigtime_t kCacheExpire = 100000; + + static void switch_to_mode(void) { @@ -182,6 +185,12 @@ scheduler_mode_operations gSchedulerLowLatencyMode = { true, + 2000, + 700, + { 2, 30 }, + + 60000, + switch_to_mode, set_cpu_enabled, has_cache_expired, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index aa960e49cd..9613a356ce 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -13,6 +13,8 @@ using namespace Scheduler; +const bigtime_t kCacheExpire = 100000; + static int32 sSmallTaskCore; @@ -243,6 +245,12 @@ scheduler_mode_operations gSchedulerPowerSavingMode = { false, + 3000, + 1000, + { 3, 60 }, + + 200000, + switch_to_mode, set_cpu_enabled, has_cache_expired, diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 9ec5d217d2..ba13a2e54e 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -611,7 +611,7 @@ should_cancel_penalty(Thread* thread) return atomic_get(&gCoreEntries[core].fStarvationCounter) != thread->scheduler_data->went_sleep_count && system_time() - thread->scheduler_data->went_sleep - > kThreadQuantum; + > sCurrentMode->base_quantum; } @@ -845,7 +845,9 @@ quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); // too little time left, it's better make the next quantum a bit longer - if (wasPreempted || schedulerThreadData->time_left <= kThreadQuantum / 50) { + if (wasPreempted + || schedulerThreadData->time_left <= sCurrentMode->minimal_quantum) { + schedulerThreadData->stolen_time += schedulerThreadData->time_left; schedulerThreadData->time_left = 0; } @@ -872,15 +874,21 @@ get_base_quantum(Thread* thread) { int32 priority = get_effective_priority(thread); + const bigtime_t kQuantum0 = sCurrentMode->base_quantum; if (priority >= B_URGENT_DISPLAY_PRIORITY) - return kThreadQuantum; + return kQuantum0; + + const bigtime_t kQuantum1 + = kQuantum0 * sCurrentMode->quantum_multipliers[0]; if (priority > B_NORMAL_PRIORITY) { - return quantum_linear_interpolation(kThreadQuantum * 2, - kThreadQuantum, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, - priority); + return quantum_linear_interpolation(kQuantum1, kQuantum0, + B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, priority); } - return quantum_linear_interpolation(kThreadQuantum * 30, - kThreadQuantum * 2, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); + + const bigtime_t kQuantum2 + = kQuantum0 * sCurrentMode->quantum_multipliers[1]; + return quantum_linear_interpolation(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, + B_IDLE_PRIORITY, priority); } @@ -902,10 +910,8 @@ compute_quantum(Thread* thread) == gCPUToCore[smp_get_current_cpu()]); CoreEntry* core = &gCoreEntries[schedulerThreadData->previous_core]; int32 threadCount = (core->fThreadCount + 1) / core->fCPUCount; - if (threadCount > 1) { - quantum = max_c(min_c(kMaximumLatency / threadCount, quantum), - kThreadQuantum / 3); - } + quantum = max_c(min_c(sCurrentMode->maximum_latency / threadCount, quantum), + sCurrentMode->minimal_quantum); schedulerThreadData->time_left = quantum; schedulerThreadData->quantum_start = system_time(); @@ -1312,7 +1318,7 @@ scheduler_on_thread_init(Thread* thread) if (thread_is_idle_thread(thread)) { static int32 sIdleThreadsID; - int32 cpu = atomic_add(&gIdleThreadsID, 1); + int32 cpu = atomic_add(&sIdleThreadsID, 1); thread->previous_cpu = &gCPU[cpu]; thread->pinned_to_cpu = 1; @@ -1421,7 +1427,6 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) // core has been disabled ASSERT(!enabled); - int32 load = CoreLoadHeap::GetKey(core); if (core->fHighLoad) { gCoreHighLoadHeap->ModifyKey(core, -1); ASSERT(gCoreHighLoadHeap->PeekMinimum() == core); @@ -1833,7 +1838,9 @@ _user_estimate_max_scheduling_latency(thread_id id) / get_effective_priority(thread); } - return max_c(threadCount * kThreadQuantum, kThreadQuantum / 5); + return min_c(max_c(threadCount * sCurrentMode->base_quantum, + sCurrentMode->minimal_quantum), + sCurrentMode->maximum_latency); } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 4c63c9fe9b..3069fffa5a 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -29,11 +29,6 @@ namespace Scheduler { -const bigtime_t kThreadQuantum = 2000; -const bigtime_t kMaximumLatency = 600000; - -const bigtime_t kCacheExpire = 100000; - const int kLowLoad = kMaxLoad * 20 / 100; const int kTargetLoad = kMaxLoad * 55 / 100; const int kHighLoad = kMaxLoad * 70 / 100; diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index b413a6bdba..2d790a3967 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -15,6 +15,12 @@ struct scheduler_mode_operations { bool avoid_boost; + bigtime_t base_quantum; + bigtime_t minimal_quantum; + bigtime_t quantum_multipliers[2]; + + bigtime_t maximum_latency; + void (*switch_to_mode)(void); void (*set_cpu_enabled)(int32 cpu, bool enabled); bool (*has_cache_expired)(Thread* thread); From 673f08a995666c3e818bf0f936152f34f8b1e81e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 05:30:50 +0100 Subject: [PATCH 166/273] kernel: Make thread_block_locked() private --- headers/private/kernel/thread.h | 33 --------------------------------- src/system/kernel/thread.cpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 61c66387d2..a5e44d2b42 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -332,39 +332,6 @@ thread_prepare_to_block(Thread* thread, uint32 flags, uint32 type, } -/*! Blocks the current thread. - - The thread is blocked until someone else unblock it. Must be called after a - call to thread_prepare_to_block(). If the thread has already been unblocked - after the previous call to thread_prepare_to_block(), this function will - return immediately. Cf. the documentation of thread_prepare_to_block() for - more details. - - The caller must hold the scheduler lock. - - \param thread The current thread. - \return The error code passed to the unblocking function. thread_interrupt() - uses \c B_INTERRUPTED. By convention \c B_OK means that the wait was - successful while another error code indicates a failure (what that means - depends on the client code). -*/ -static inline status_t -thread_block_locked(Thread* thread) -{ - if (thread->wait.status == 1) { - // check for signals, if interruptible - if (thread_is_interrupted(thread, thread->wait.flags)) { - thread->wait.status = B_INTERRUPTED; - } else { - thread->next_state = B_THREAD_WAITING; - scheduler_reschedule(); - } - } - - return thread->wait.status; -} - - /*! Unblocks the specified blocked thread. If the thread is no longer waiting (e.g. because thread_unblock_locked() has diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index f3474b58ca..141486709f 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -2805,6 +2805,39 @@ thread_block_timeout(timer* timer) } +/*! Blocks the current thread. + + The thread is blocked until someone else unblock it. Must be called after a + call to thread_prepare_to_block(). If the thread has already been unblocked + after the previous call to thread_prepare_to_block(), this function will + return immediately. Cf. the documentation of thread_prepare_to_block() for + more details. + + The caller must hold the scheduler lock. + + \param thread The current thread. + \return The error code passed to the unblocking function. thread_interrupt() + uses \c B_INTERRUPTED. By convention \c B_OK means that the wait was + successful while another error code indicates a failure (what that means + depends on the client code). +*/ +static inline status_t +thread_block_locked(Thread* thread) +{ + if (thread->wait.status == 1) { + // check for signals, if interruptible + if (thread_is_interrupted(thread, thread->wait.flags)) { + thread->wait.status = B_INTERRUPTED; + } else { + thread->next_state = B_THREAD_WAITING; + scheduler_reschedule(); + } + } + + return thread->wait.status; +} + + /*! Blocks the current thread. The function acquires the scheduler lock and calls thread_block_locked(). From c19f1e4fbb379d6a6456d6cbaa4f09033274de18 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 18:09:56 +0100 Subject: [PATCH 167/273] scheduler: Add missing check for real time threads --- src/system/kernel/scheduler/scheduler.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ba13a2e54e..6fbe308f13 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -903,6 +903,9 @@ compute_quantum(Thread* thread) else quantum = get_base_quantum(thread); + if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return quantum; + quantum += schedulerThreadData->stolen_time; schedulerThreadData->stolen_time = 0; From 2b7ea4cddf362859cd3089e3cb95481cda4adc5f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 19:31:10 +0100 Subject: [PATCH 168/273] kernel: Remove Thread::next_state --- headers/private/kernel/kscheduler.h | 4 ++-- headers/private/kernel/thread.h | 2 +- headers/private/kernel/thread_types.h | 1 - src/system/kernel/arch/x86/arch_int.cpp | 2 +- src/system/kernel/image.cpp | 3 +-- src/system/kernel/scheduler/scheduler.cpp | 19 +++++++++---------- src/system/kernel/signal.cpp | 3 +-- src/system/kernel/team.cpp | 3 +-- src/system/kernel/thread.cpp | 18 +++++++----------- 9 files changed, 23 insertions(+), 32 deletions(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index cbad85d9e2..16316e87c1 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -43,7 +43,7 @@ void scheduler_reschedule_ici(void); indefinitely, the function will eventually return. The caller must hold the current thread \c scheduler_lock. */ -void scheduler_reschedule(void); +void scheduler_reschedule(int32 next_state); /*! Sets the given thread's priority. The thread may be running or may be in the ready-to-run queue. @@ -112,7 +112,7 @@ static inline void scheduler_reschedule_if_necessary_locked() { if (gCPU[smp_get_current_cpu()].invoke_scheduler) - scheduler_reschedule(); + scheduler_reschedule(B_THREAD_READY); } diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index a5e44d2b42..96d80ba912 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -211,7 +211,7 @@ thread_is_interrupted(Thread* thread, uint32 flags) static inline bool thread_is_blocked(Thread* thread) { - return thread->wait.status == 1; + return atomic_get(&thread->wait.status) == 1; } diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 6212deb48d..2a7051bc3b 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -424,7 +424,6 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, int32 priority; // protected by scheduler lock int32 io_priority; // protected by fLock int32 state; // protected by scheduler lock - int32 next_state; // protected by scheduler lock struct cpu_ent *cpu; // protected by scheduler lock struct cpu_ent *previous_cpu; // protected by scheduler lock int32 pinned_to_cpu; // only accessed by this thread or in the diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index abcaf5fa08..41176d850f 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -232,7 +232,7 @@ x86_hardware_interrupt(struct iframe* frame) cpu_status state = disable_interrupts(); if (thread->cpu->invoke_scheduler) { SpinLocker schedulerLocker(thread->scheduler_lock); - scheduler_reschedule(); + scheduler_reschedule(B_THREAD_READY); schedulerLocker.Unlock(); restore_interrupts(state); } else if (thread->post_interrupt_callback != NULL) { diff --git a/src/system/kernel/image.cpp b/src/system/kernel/image.cpp index 55ac115dff..c948b69f86 100644 --- a/src/system/kernel/image.cpp +++ b/src/system/kernel/image.cpp @@ -383,8 +383,7 @@ notify_loading_app(status_t result, bool suspend) Thread* thread = thread_get_current_thread(); InterruptsSpinLocker schedulerLocker(thread->scheduler_lock); - thread->next_state = B_THREAD_SUSPENDED; - scheduler_reschedule(); + scheduler_reschedule(B_THREAD_SUSPENDED); } } } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 6fbe308f13..109c2821b2 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -620,7 +620,7 @@ enqueue(Thread* thread, bool newOne) { ASSERT(thread != NULL); - thread->state = thread->next_state = B_THREAD_READY; + thread->state = B_THREAD_READY; compute_thread_load(thread); @@ -1148,7 +1148,7 @@ update_thread_times(Thread* oldThread, Thread* nextThread) static void -reschedule(void) +reschedule(int32 nextState) { ASSERT(!are_interrupts_enabled()); @@ -1162,7 +1162,7 @@ reschedule(void) TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, oldThread->id); - oldThread->state = oldThread->next_state; + oldThread->state = nextState; scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; // return time spent in interrupts @@ -1172,7 +1172,7 @@ reschedule(void) bool enqueueOldThread = false; bool putOldThreadAtBack = false; - switch (oldThread->next_state) { + switch (nextState) { case B_THREAD_RUNNING: case B_THREAD_READY: enqueueOldThread = true; @@ -1197,7 +1197,7 @@ reschedule(void) increase_penalty(oldThread); thread_goes_away(oldThread); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", - oldThread->id, oldThread->next_state); + oldThread->id, nextState); break; } @@ -1247,7 +1247,6 @@ reschedule(void) update_cpu_priority(thisCPU, get_effective_priority(nextThread)); nextThread->state = B_THREAD_RUNNING; - nextThread->next_state = B_THREAD_READY; ASSERT(nextThread->scheduler_data->previous_core == thisCore); @@ -1291,16 +1290,16 @@ reschedule(void) Note: expects thread spinlock to be held */ void -scheduler_reschedule(void) +scheduler_reschedule(int32 nextState) { if (!sSchedulerEnabled) { Thread* thread = thread_get_current_thread(); - if (thread != NULL && thread->next_state != B_THREAD_READY) + if (thread != NULL && nextState != B_THREAD_READY) panic("scheduler_reschedule_no_op() called in non-ready thread"); return; } - reschedule(); + reschedule(nextState); } @@ -1346,7 +1345,7 @@ scheduler_start(void) { InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); - reschedule(); + reschedule(B_THREAD_READY); } diff --git a/src/system/kernel/signal.cpp b/src/system/kernel/signal.cpp index 5377903308..34af28a51f 100644 --- a/src/system/kernel/signal.cpp +++ b/src/system/kernel/signal.cpp @@ -1129,8 +1129,7 @@ handle_signals(Thread* thread) if (!resume) { InterruptsSpinLocker _(thread->scheduler_lock); - thread->next_state = B_THREAD_SUSPENDED; - scheduler_reschedule(); + scheduler_reschedule(B_THREAD_SUSPENDED); } continue; diff --git a/src/system/kernel/team.cpp b/src/system/kernel/team.cpp index 175645a15c..dde114ed77 100644 --- a/src/system/kernel/team.cpp +++ b/src/system/kernel/team.cpp @@ -1811,8 +1811,7 @@ load_image_internal(char**& _flatArgs, size_t flatArgsSize, int32 argCount, Thread* thread = thread_get_current_thread(); InterruptsSpinLocker schedulerLocker(thread->scheduler_lock); - thread->next_state = B_THREAD_SUSPENDED; - scheduler_reschedule(); + scheduler_reschedule(B_THREAD_SUSPENDED); } if (loadingInfo.result < B_OK) diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index 141486709f..fc6cbe3842 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -892,7 +892,6 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel) thread->priority = attributes.priority == -1 ? B_NORMAL_PRIORITY : attributes.priority; thread->state = B_THREAD_SUSPENDED; - thread->next_state = B_THREAD_SUSPENDED; thread->sig_block_mask = attributes.signal_mask; @@ -1501,7 +1500,9 @@ make_thread_suspended(int argc, char **argv) if (thread->id != id) continue; - thread->next_state = B_THREAD_SUSPENDED; + Signal signal(SIGSTOP, SI_USER, B_OK, team_get_kernel_team()->id); + send_signal_to_thread(thread, signal, B_DO_NOT_RESCHEDULE); + kprintf("thread %" B_PRId32 " suspended\n", id); found = true; break; @@ -1710,7 +1711,6 @@ _dump_thread_info(Thread *thread, bool shortInfo) kprintf("priority: %" B_PRId32 " (I/O: %" B_PRId32 ")\n", thread->priority, thread->io_priority); kprintf("state: %s\n", state_to_text(thread, thread->state)); - kprintf("next_state: %s\n", state_to_text(thread, thread->next_state)); kprintf("cpu: %p ", thread->cpu); if (thread->cpu) kprintf("(%d)\n", thread->cpu->cpu_num); @@ -2250,8 +2250,7 @@ thread_exit(void) sUndertakerCondition.NotifyOne(); undertakerLocker.Unlock(); - thread->next_state = THREAD_STATE_FREE_ON_RESCHED; - scheduler_reschedule(); + scheduler_reschedule(THREAD_STATE_FREE_ON_RESCHED); panic("never can get here\n"); } @@ -2399,7 +2398,7 @@ thread_yield(void) InterruptsSpinLocker _(thread->scheduler_lock); thread->has_yielded = true; - scheduler_reschedule(); + scheduler_reschedule(B_THREAD_READY); } @@ -2678,7 +2677,6 @@ thread_init(kernel_args *args) thread->team = team_get_kernel_team(); thread->priority = B_IDLE_PRIORITY; thread->state = B_THREAD_RUNNING; - thread->next_state = B_THREAD_READY; sprintf(name, "idle thread %" B_PRIu32 " kstack", i + 1); thread->kernel_stack_area = find_area(name); @@ -2828,10 +2826,8 @@ thread_block_locked(Thread* thread) // check for signals, if interruptible if (thread_is_interrupted(thread, thread->wait.flags)) { thread->wait.status = B_INTERRUPTED; - } else { - thread->next_state = B_THREAD_WAITING; - scheduler_reschedule(); - } + } else + scheduler_reschedule(B_THREAD_WAITING); } return thread->wait.status; From 255b6017502aa4548d14bff5d0b366b0f266b833 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 29 Nov 2013 20:32:00 +0100 Subject: [PATCH 169/273] scheduler: Do not send reschedule ICI when not needed --- src/system/kernel/scheduler/scheduler.cpp | 61 +++++++++++++++-------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 109c2821b2..6f71982e58 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -449,12 +449,7 @@ static inline void update_cpu_priority(int32 cpu, int32 priority) { int32 core = gCPUToCore[cpu]; - - SpinLocker coreLocker(gCoreEntries[core].fCPULock); - int32 corePriority = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); - - gCPUEntries[cpu].fPriority = priority; gCPUPriorityHeaps[core].ModifyKey(&gCPUEntries[cpu], priority); if (gSingleCore) @@ -511,29 +506,41 @@ choose_core(Thread* thread) static inline int32 -choose_cpu(int32 core) +choose_cpu(int32 core, Thread* thread, bool& rescheduleNeeded) { SpinLocker cpuLocker(gCoreEntries[core].fCPULock); CPUEntry* entry = gCPUPriorityHeaps[core].PeekMinimum(); ASSERT(entry != NULL); + + int32 threadPriority = get_effective_priority(thread); + if (CPUHeap::GetKey(entry) < threadPriority) { + update_cpu_priority(entry->fCPUNumber, threadPriority); + rescheduleNeeded = true; + } else + rescheduleNeeded = false; + return entry->fCPUNumber; } -static void +static bool choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) { + bool rescheduleNeeded = false; + if (targetCore == -1 && targetCPU != -1) targetCore = gCPUToCore[targetCPU]; else if (targetCore != -1 && targetCPU == -1) - targetCPU = choose_cpu(targetCore); + targetCPU = choose_cpu(targetCore, thread, rescheduleNeeded); else if (targetCore == -1 && targetCPU == -1) { targetCore = choose_core(thread); - targetCPU = choose_cpu(targetCore); + targetCPU = choose_cpu(targetCore, thread, rescheduleNeeded); } ASSERT(targetCore >= 0 && targetCore < gCoreCount); ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); + + return rescheduleNeeded; } @@ -644,7 +651,7 @@ enqueue(Thread* thread, bool newOne) targetCore = schedulerThreadData->previous_core; } - choose_core_and_cpu(thread, targetCore, targetCPU); + bool rescheduleNeeded = choose_core_and_cpu(thread, targetCore, targetCPU); schedulerThreadData->previous_core = targetCore; TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", @@ -666,13 +673,10 @@ enqueue(Thread* thread, bool newOne) NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - int32 targetPriority = gCPUEntries[targetCPU].fPriority; - - if (threadPriority > targetPriority) { - // It is possible that another CPU schedules the thread before the - // target CPU. However, since the target CPU is sent an ICI it will - // reschedule anyway and update its heap key to the correct value. - update_cpu_priority(targetCPU, threadPriority); + int32 heapPriority = CPUHeap::GetKey(&gCPUEntries[targetCPU]); + if (threadPriority > atomic_get(&gCPUEntries[targetCPU].fPriority) + && (threadPriority > heapPriority + || (threadPriority == heapPriority && rescheduleNeeded))) { if (targetCPU == smp_get_current_cpu()) gCPU[targetCPU].invoke_scheduler = true; @@ -752,9 +756,18 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) thread->priority = priority; + int32 previousCore = thread->scheduler_data->previous_core; + ASSERT(previousCore >= 0); + if (thread->state != B_THREAD_READY) { - if (thread->state == B_THREAD_RUNNING) + if (thread->state == B_THREAD_RUNNING) { + ASSERT(thread->previous_cpu != NULL); + + SpinLocker coreLocker(gCoreEntries[previousCore].fCPULock); + + gCPUEntries[thread->cpu->cpu_num].fPriority = priority; update_cpu_priority(thread->cpu->cpu_num, priority); + } return oldPriority; } @@ -767,8 +780,6 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) ASSERT(thread->previous_cpu != NULL); previousCPU = thread->previous_cpu->cpu_num; } - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); SpinLocker runQueueLocker(gCoreEntries[previousCore].fQueueLock); @@ -913,6 +924,8 @@ compute_quantum(Thread* thread) == gCPUToCore[smp_get_current_cpu()]); CoreEntry* core = &gCoreEntries[schedulerThreadData->previous_core]; int32 threadCount = (core->fThreadCount + 1) / core->fCPUCount; + threadCount = max_c(threadCount, 1); + quantum = max_c(min_c(sCurrentMode->maximum_latency / threadCount, quantum), sCurrentMode->minimal_quantum); @@ -1222,6 +1235,9 @@ reschedule(int32 nextState) putOldThreadAtBack); } + atomic_set(&gCPUEntries[thisCPU].fPriority, + get_effective_priority(nextThread)); + if (nextThread != oldThread) { if (enqueueOldThread) { if (putOldThreadAtBack) @@ -1243,8 +1259,10 @@ reschedule(int32 nextState) oldThread, nextThread); // update CPU heap - if (!gCPU[thisCPU].disabled) + if (!gCPU[thisCPU].disabled) { + SpinLocker coreLocker(gCoreEntries[thisCore].fCPULock); update_cpu_priority(thisCPU, get_effective_priority(nextThread)); + } nextThread->state = B_THREAD_RUNNING; @@ -1421,6 +1439,7 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) if (enabled) core->fCPUCount++; else { + gCPUEntries[cpu].fPriority = B_IDLE_PRIORITY; update_cpu_priority(cpu, B_IDLE_PRIORITY); core->fCPUCount--; } From 2e3cbcfa8a455f6d0ac65f51e795e42ccfbf9eb3 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 2 Dec 2013 03:19:25 +0100 Subject: [PATCH 170/273] x86: Allocate as much initial physical page pools as needed --- .../x86/paging/32bit/X86PagingMethod32Bit.cpp | 51 ++++++++++++----- .../x86/paging/32bit/X86PagingMethod32Bit.h | 2 + .../x86/paging/pae/X86PagingMethodPAE.cpp | 55 +++++++++++++------ .../arch/x86/paging/pae/X86PagingMethodPAE.h | 2 + .../x86_physical_page_mapper_large_memory.cpp | 30 +++++----- .../x86_physical_page_mapper_large_memory.h | 12 +++- 6 files changed, 106 insertions(+), 46 deletions(-) diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp index be91ed524a..0f0272f6ba 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -36,6 +37,10 @@ #endif +#define MAX_INITIAL_POOLS \ + ((SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS + 1023) / 1024) + + using X86LargePhysicalPageMapper::PhysicalPageSlot; @@ -60,7 +65,7 @@ public: addr_t virtualAddress); public: - static PhysicalPageSlotPool sInitialPhysicalPagePool; + static PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS]; private: area_id fDataArea; @@ -71,7 +76,8 @@ private: X86PagingMethod32Bit::PhysicalPageSlotPool - X86PagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool; + X86PagingMethod32Bit::PhysicalPageSlotPool::sInitialPhysicalPagePool[ + MAX_INITIAL_POOLS]; X86PagingMethod32Bit::PhysicalPageSlotPool::~PhysicalPageSlotPool() @@ -287,20 +293,23 @@ X86PagingMethod32Bit::Init(kernel_args* args, X86PagingStructures32Bit::StaticInit(); - // create the initial pool for the physical page mapper + // create the initial pools for the physical page mapper + int32 poolCount = _GetInitialPoolCount(); PhysicalPageSlotPool* pool = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) - PhysicalPageSlotPool; - status_t error = pool->InitInitial(args); - if (error != B_OK) { - panic("X86PagingMethod32Bit::Init(): Failed to create initial pool " - "for physical page mapper!"); - return error; + PhysicalPageSlotPool[poolCount]; + for (int32 i = 0; i < poolCount; i++) { + status_t error = pool[i].InitInitial(args); + if (error != B_OK) { + panic("X86PagingMethod32Bit::Init(): Failed to create initial pool " + "for physical page mapper!"); + return error; + } } // create physical page mapper - large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper, - fKernelPhysicalPageMapper); + large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool), + fPhysicalPageMapper, fKernelPhysicalPageMapper); // TODO: Select the best page mapper! // enable global page feature if available @@ -337,10 +346,13 @@ X86PagingMethod32Bit::InitPostArea(kernel_args* args) if (area < B_OK) return area; - error = PhysicalPageSlotPool::sInitialPhysicalPagePool - .InitInitialPostArea(args); - if (error != B_OK) - return error; + int32 poolCount = _GetInitialPoolCount(); + for (int32 i = 0; i < poolCount; i++) { + status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i] + .InitInitialPostArea(args); + if (error != B_OK) + return error; + } return B_OK; } @@ -517,6 +529,15 @@ X86PagingMethod32Bit::PutPageTableEntryInTable(page_table_entry* entry, } +inline int32 +X86PagingMethod32Bit::_GetInitialPoolCount() +{ + int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU + + EXTRA_SLOTS; + return (requiredSlots + 1023) / 1024; +} + + /*static*/ void X86PagingMethod32Bit::_EarlyPreparePageTables(page_table_entry* pageTables, addr_t address, size_t size) diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h index 8df3181552..96cb50605c 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.h @@ -80,6 +80,8 @@ private: friend struct PhysicalPageSlotPool; private: + inline int32 _GetInitialPoolCount(); + static void _EarlyPreparePageTables( page_table_entry* pageTables, addr_t address, size_t size); diff --git a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp index e4ee9bccb0..e96b0be792 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp +++ b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp @@ -15,6 +15,7 @@ #include +#include #include #include #include @@ -40,6 +41,11 @@ #if B_HAIKU_PHYSICAL_BITS == 64 +#define MAX_INITIAL_POOLS \ + ((SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS \ + + kPAEPageTableEntryCount - 1) / kPAEPageTableEntryCount) + + using X86LargePhysicalPageMapper::PhysicalPageSlot; @@ -364,7 +370,7 @@ public: addr_t virtualAddress); public: - static PhysicalPageSlotPool sInitialPhysicalPagePool; + static PhysicalPageSlotPool sInitialPhysicalPagePool[MAX_INITIAL_POOLS]; private: area_id fDataArea; @@ -375,7 +381,8 @@ private: X86PagingMethodPAE::PhysicalPageSlotPool - X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool; + X86PagingMethodPAE::PhysicalPageSlotPool::sInitialPhysicalPagePool[ + MAX_INITIAL_POOLS]; X86PagingMethodPAE::PhysicalPageSlotPool::~PhysicalPageSlotPool() @@ -585,20 +592,23 @@ X86PagingMethodPAE::Init(kernel_args* args, fEarlyPageStructuresSize, fKernelVirtualPageDirs, fKernelPhysicalPageDirs, fFreeVirtualSlot, fFreeVirtualSlotPTE); - // create the initial pool for the physical page mapper + // create the initial pools for the physical page mapper + int32 poolCount = _GetInitialPoolCount(); PhysicalPageSlotPool* pool = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) - PhysicalPageSlotPool; - status_t error = pool->InitInitial(this, args); - if (error != B_OK) { - panic("X86PagingMethodPAE::Init(): Failed to create initial pool " - "for physical page mapper!"); - return error; + PhysicalPageSlotPool[poolCount]; + for (int32 i = 0; i < poolCount; i++) { + status_t error = pool[i].InitInitial(this, args); + if (error != B_OK) { + panic("X86PagingMethodPAE::Init(): Failed to create initial pool " + "for physical page mapper!"); + return error; + } } // create physical page mapper - large_memory_physical_page_ops_init(args, pool, fPhysicalPageMapper, - fKernelPhysicalPageMapper); + large_memory_physical_page_ops_init(args, pool, poolCount, sizeof(*pool), + fPhysicalPageMapper, fKernelPhysicalPageMapper); *_physicalPageMapper = fPhysicalPageMapper; return B_OK; @@ -615,11 +625,14 @@ X86PagingMethodPAE::InitPostArea(kernel_args* args) if (area < B_OK) return area; - // let the initial page pool create areas for its structures - status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool - .InitInitialPostArea(args); - if (error != B_OK) - return error; + // let the initial page pools create areas for its structures + int32 poolCount = _GetInitialPoolCount(); + for (int32 i = 0; i < poolCount; i++) { + status_t error = PhysicalPageSlotPool::sInitialPhysicalPagePool[i] + .InitInitialPostArea(args); + if (error != B_OK) + return error; + } // The early physical page mapping mechanism is no longer needed. Unmap the // slot. @@ -884,6 +897,16 @@ X86PagingMethodPAE::Free32BitPage(void* address, phys_addr_t physicalAddress, } +inline int32 +X86PagingMethodPAE::_GetInitialPoolCount() +{ + int32 requiredSlots = smp_get_num_cpus() * TOTAL_SLOTS_PER_CPU + + EXTRA_SLOTS; + return (requiredSlots + kPAEPageTableEntryCount - 1) + / kPAEPageTableEntryCount; +} + + bool X86PagingMethodPAE::_EarlyQuery(addr_t virtualAddress, phys_addr_t* _physicalAddress) diff --git a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h index c787bdc0cb..44715e541b 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h +++ b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.h @@ -100,6 +100,8 @@ private: friend struct PhysicalPageSlotPool; private: + inline int32 _GetInitialPoolCount(); + bool _EarlyQuery(addr_t virtualAddress, phys_addr_t* _physicalAddress); pae_page_table_entry* _EarlyGetPageTable(phys_addr_t address); diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp index b317f92e28..0b75b14dc4 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp @@ -48,12 +48,6 @@ // a little longer, thus avoiding re-mapping. #define SLOTS_PER_TRANSLATION_MAP 4 -#define USER_SLOTS_PER_CPU 16 -#define KERNEL_SLOTS_PER_CPU 16 -#define TOTAL_SLOTS_PER_CPU (USER_SLOTS_PER_CPU \ - + KERNEL_SLOTS_PER_CPU + 1) - // one slot is for use in interrupts - using X86LargePhysicalPageMapper::PhysicalPageSlot; using X86LargePhysicalPageMapper::PhysicalPageSlotPool; @@ -125,9 +119,10 @@ public: LargeMemoryPhysicalPageMapper(); status_t Init(kernel_args* args, - PhysicalPageSlotPool* initialPool, - TranslationMapPhysicalPageMapper*& - _kernelPageMapper); + PhysicalPageSlotPool* initialPools, + int32 initalPoolCount, size_t poolSize, + TranslationMapPhysicalPageMapper*& + _kernelPageMapper); virtual status_t CreateTranslationMapPhysicalPageMapper( TranslationMapPhysicalPageMapper** _mapper); @@ -428,11 +423,16 @@ LargeMemoryPhysicalPageMapper::LargeMemoryPhysicalPageMapper() status_t LargeMemoryPhysicalPageMapper::Init(kernel_args* args, - PhysicalPageSlotPool* initialPool, + PhysicalPageSlotPool* initialPools, int32 initialPoolCount, size_t poolSize, TranslationMapPhysicalPageMapper*& _kernelPageMapper) { - fInitialPool = initialPool; - fNonEmptyPools.Add(fInitialPool); + ASSERT(initialPoolCount >= 1); + + fInitialPool = initialPools; + for (int32 i = 0; i < initialPoolCount; i++) { + uint8* pointer = (uint8*)initialPools + i * poolSize; + fNonEmptyPools.Add((PhysicalPageSlotPool*)pointer); + } // get the debug slot GetSlot(true, fDebugSlot); @@ -755,12 +755,14 @@ LargeMemoryPhysicalPageMapper::GetSlotQueue(int32 cpu, bool user) status_t large_memory_physical_page_ops_init(kernel_args* args, - X86LargePhysicalPageMapper::PhysicalPageSlotPool* initialPool, + X86LargePhysicalPageMapper::PhysicalPageSlotPool* initialPools, + int32 initialPoolCount, size_t poolSize, X86PhysicalPageMapper*& _pageMapper, TranslationMapPhysicalPageMapper*& _kernelPageMapper) { new(&sPhysicalPageMapper) LargeMemoryPhysicalPageMapper; - sPhysicalPageMapper.Init(args, initialPool, _kernelPageMapper); + sPhysicalPageMapper.Init(args, initialPools, initialPoolCount, poolSize, + _kernelPageMapper); _pageMapper = &sPhysicalPageMapper; return B_OK; diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.h b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.h index 93986fb15f..8e695417cb 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.h +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.h @@ -11,6 +11,15 @@ #include +#define USER_SLOTS_PER_CPU 16 +#define KERNEL_SLOTS_PER_CPU 16 +#define TOTAL_SLOTS_PER_CPU (USER_SLOTS_PER_CPU \ + + KERNEL_SLOTS_PER_CPU + 1) + // one slot is for use in interrupts + +#define EXTRA_SLOTS 2 + + class TranslationMapPhysicalPageMapper; class X86PhysicalPageMapper; struct kernel_args; @@ -53,7 +62,8 @@ protected: status_t large_memory_physical_page_ops_init(kernel_args* args, - X86LargePhysicalPageMapper::PhysicalPageSlotPool* initialPool, + X86LargePhysicalPageMapper::PhysicalPageSlotPool* initialPools, + int32 initialPoolCount, size_t poolSize, X86PhysicalPageMapper*& _pageMapper, TranslationMapPhysicalPageMapper*& _kernelPageMapper); From d287274dcec634da4973a1b92c97dd14d7c5ecd0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 5 Dec 2013 22:47:30 +0100 Subject: [PATCH 171/273] scheduler: Code refactoring --- headers/private/kernel/thread_types.h | 11 +- headers/private/kernel/util/atomic.h | 2 + src/system/kernel/Jamfile | 2 + src/system/kernel/scheduler/low_latency.cpp | 74 +- src/system/kernel/scheduler/power_saving.cpp | 137 +- src/system/kernel/scheduler/scheduler.cpp | 1396 ++++------------- .../kernel/scheduler/scheduler_common.h | 176 +-- src/system/kernel/scheduler/scheduler_cpu.cpp | 466 ++++++ src/system/kernel/scheduler/scheduler_cpu.h | 170 ++ src/system/kernel/scheduler/scheduler_modes.h | 38 +- .../kernel/scheduler/scheduler_thread.cpp | 158 ++ .../kernel/scheduler/scheduler_thread.h | 360 +++++ 12 files changed, 1555 insertions(+), 1435 deletions(-) create mode 100644 src/system/kernel/scheduler/scheduler_cpu.cpp create mode 100644 src/system/kernel/scheduler/scheduler_cpu.h create mode 100644 src/system/kernel/scheduler/scheduler_thread.cpp create mode 100644 src/system/kernel/scheduler/scheduler_thread.h diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 2a7051bc3b..eb7f1ad5e1 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -58,12 +57,15 @@ struct cpu_ent; struct image; // defined in image.c struct io_context; struct realtime_sem_context; // defined in realtime_sem.cpp -struct scheduler_thread_data; struct select_info; struct user_thread; // defined in libroot/user_thread.h struct VMAddressSpace; struct xsi_sem_context; // defined in xsi_semaphore.cpp +namespace Scheduler { + struct ThreadData; +} + namespace BKernel { struct Team; struct Thread; @@ -412,8 +414,7 @@ private: }; -struct Thread : TeamThreadIteratorEntry, KernelReferenceable, - RunQueueLinkImpl { +struct Thread : TeamThreadIteratorEntry, KernelReferenceable { int32 flags; // summary of events relevant in interrupt // handlers (signals pending, user debugging // enabled, etc.) @@ -444,7 +445,7 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, bool in_kernel; // protected by time_lock, only written by // this thread bool has_yielded; // protected by scheduler lock - struct scheduler_thread_data* scheduler_data; // protected by scheduler lock + Scheduler::ThreadData* scheduler_data; // protected by scheduler lock struct user_thread* user_thread; // write-protected by fLock, only // modified by the thread itself and diff --git a/headers/private/kernel/util/atomic.h b/headers/private/kernel/util/atomic.h index 14725492c8..40b057c608 100644 --- a/headers/private/kernel/util/atomic.h +++ b/headers/private/kernel/util/atomic.h @@ -10,6 +10,8 @@ #include +#include + #ifdef __cplusplus diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 4173486dff..ab80c5d31a 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -65,6 +65,8 @@ KernelMergeObject kernel_core.o : low_latency.cpp power_saving.cpp scheduler.cpp + scheduler_cpu.cpp + scheduler_thread.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 6adfe4e6fb..a272f53214 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -7,7 +7,9 @@ #include #include "scheduler_common.h" +#include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_thread.h" using namespace Scheduler; @@ -29,16 +31,12 @@ set_cpu_enabled(int32 /* cpu */, bool /* enabled */) static bool -has_cache_expired(Thread* thread) +has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; - return atomic_get64(&coreEntry->fActiveTime) - - schedulerThreadData->went_sleep_active > kCacheExpire; + return atomic_get64(&threadData->GetCore()->fActiveTime) + - threadData->fWentSleepActive > kCacheExpire; } @@ -58,51 +56,45 @@ get_most_idle_package(void) } -static int32 -choose_core(Thread* thread) +static CoreEntry* +choose_core(const ThreadData* /* threadData */) { - CoreEntry* entry = NULL; - ReadSpinLocker locker(gIdlePackageLock); // wake new package - PackageEntry* package = gIdlePackageList->Last(); + PackageEntry* package = gIdlePackageList.Last(); if (package == NULL) { // wake new core package = get_most_idle_package(); } locker.Unlock(); + CoreEntry* core = NULL; if (package != NULL) { ReadSpinLocker _(package->fCoreLock); - entry = package->fIdleCores.Last(); + core = package->fIdleCores.Last(); } - if (entry == NULL) { + if (core == NULL) { ReadSpinLocker coreLocker(gCoreHeapsLock); // no idle cores, use least occupied core - entry = gCoreLoadHeap->PeekMinimum(); - if (entry == NULL) - entry = gCoreHighLoadHeap->PeekMinimum(); + core = gCoreLoadHeap.PeekMinimum(); + if (core == NULL) + core = gCoreHighLoadHeap.PeekMinimum(); } - ASSERT(entry != NULL); - return entry->fCoreID; + ASSERT(core != NULL); + return core; } static bool -should_rebalance(Thread* thread) +should_rebalance(const ThreadData* threadData) { - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; - - int32 coreLoad = get_core_load(coreEntry); + int32 coreLoad = threadData->GetCore()->GetLoad(); // If the thread produces more than 50% of the load, leave it here. In // such situation it is better to move other threads away. - if (schedulerThreadData->load >= coreLoad / 2) + if (threadData->GetLoad() >= coreLoad / 2) return false; // If there is high load on this core but this thread does not contribute @@ -110,22 +102,20 @@ should_rebalance(Thread* thread) if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); - if (other != NULL && coreLoad - get_core_load(other) - >= kLoadDifference) { + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); + if (other != NULL && coreLoad - other->GetLoad() >= kLoadDifference) return true; - } } // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); + other = gCoreHighLoadHeap.PeekMinimum(); ASSERT(other != NULL); - return coreLoad - get_core_load(other) >= kLoadDifference * 2; + return coreLoad - other->GetLoad() >= kLoadDifference * 2; } @@ -155,26 +145,22 @@ rebalance_irqs(bool idle) return; ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); + other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; + int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; cpuLocker.Unlock(); - ASSERT(other != NULL); - int32 thisCore = gCPUToCore[smp_get_current_cpu()]; - if (other->fCoreID == thisCore) + CoreEntry* core = CoreEntry::GetCore(cpu->cpu_num); + if (other == core) return; - - if (get_core_load(other) + kLoadDifference - >= get_core_load(&gCoreEntries[thisCore])) { + if (other->GetLoad() + kLoadDifference >= core->GetLoad()) return; - } assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 9613a356ce..c1eeb61609 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -4,10 +4,13 @@ */ +#include #include #include "scheduler_common.h" +#include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_thread.h" using namespace Scheduler; @@ -15,13 +18,13 @@ using namespace Scheduler; const bigtime_t kCacheExpire = 100000; -static int32 sSmallTaskCore; +static CoreEntry* sSmallTaskCore; static void switch_to_mode(void) { - sSmallTaskCore = -1; + sSmallTaskCore = NULL; } @@ -29,34 +32,32 @@ static void set_cpu_enabled(int32 cpu, bool enabled) { if (!enabled) - sSmallTaskCore = -1; + sSmallTaskCore = NULL; } static bool -has_cache_expired(Thread* thread) +has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - return system_time() - schedulerThreadData->went_sleep > kCacheExpire; + return system_time() - threadData->fWentSleep > kCacheExpire; } -static int32 +static CoreEntry* choose_small_task_core(void) { ReadSpinLocker locker(gCoreHeapsLock); - CoreEntry* candidate = gCoreLoadHeap->PeekMaximum(); + CoreEntry* core = gCoreLoadHeap.PeekMaximum(); locker.Unlock(); - if (candidate == NULL) + + if (core == NULL) return sSmallTaskCore; - int32 core = candidate->fCoreID; - int32 smallTaskCore = atomic_test_and_set(&sSmallTaskCore, core, -1); - if (smallTaskCore == -1) + CoreEntry* smallTaskCore + = atomic_pointer_test_and_set(&sSmallTaskCore, core, (CoreEntry*)NULL); + if (smallTaskCore == NULL) return core; return smallTaskCore; } @@ -65,121 +66,118 @@ choose_small_task_core(void) static CoreEntry* choose_idle_core(void) { - PackageEntry* current = NULL; + PackageEntry* package = NULL; + for (int32 i = 0; i < gPackageCount; i++) { - if (gPackageEntries[i].fIdleCoreCount != 0 && (current == NULL - || gPackageEntries[i].fIdleCoreCount - < current->fIdleCoreCount)) { - current = &gPackageEntries[i]; + PackageEntry* current = &gPackageEntries[i]; + if (current->fIdleCoreCount != 0 && (package == NULL + || current->fIdleCoreCount < package->fIdleCoreCount)) { + package = current; } } - if (current == NULL) { + if (package == NULL) { ReadSpinLocker _(gIdlePackageLock); - current = gIdlePackageList->Last(); + package = gIdlePackageList.Last(); } - if (current != NULL) { - ReadSpinLocker _(current->fCoreLock); - return current->fIdleCores.Last(); + if (package != NULL) { + ReadSpinLocker _(package->fCoreLock); + return package->fIdleCores.Last(); } return NULL; } -static int32 -choose_core(Thread* thread) +static CoreEntry* +choose_core(const ThreadData* threadData) { - CoreEntry* entry; + CoreEntry* core = NULL; - int32 core = -1; // try to pack all threads on one core core = choose_small_task_core(); - if (core != -1 - && get_core_load(&gCoreEntries[core]) + thread->scheduler_data->load - < kHighLoad) { - entry = &gCoreEntries[core]; - } else { + if (core == NULL || core->GetLoad() + threadData->GetLoad() >= kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); + // run immediately on already woken core - entry = gCoreLoadHeap->PeekMinimum(); - if (entry == NULL) { + core = gCoreLoadHeap.PeekMinimum(); + if (core == NULL) { coreLocker.Unlock(); - entry = choose_idle_core(); + core = choose_idle_core(); - if (entry == NULL) { + if (core == NULL) { coreLocker.Lock(); - entry = gCoreHighLoadHeap->PeekMinimum(); + core = gCoreHighLoadHeap.PeekMinimum(); } } } - ASSERT(entry != NULL); - return entry->fCoreID; + ASSERT(core != NULL); + return core; } static bool -should_rebalance(Thread* thread) +should_rebalance(const ThreadData* threadData) { ASSERT(!gSingleCore); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); + CoreEntry* core = threadData->GetCore(); - int32 core = schedulerThreadData->previous_core; - CoreEntry* coreEntry = &gCoreEntries[core]; - - int32 coreLoad = get_core_load(coreEntry); + int32 coreLoad = core->GetLoad(); if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { - sSmallTaskCore = -1; + sSmallTaskCore = NULL; choose_small_task_core(); - if (schedulerThreadData->load > coreLoad / 3) + + if (threadData->GetLoad() > coreLoad / 3) return false; return coreLoad > kVeryHighLoad; } - if (schedulerThreadData->load >= coreLoad / 2) + if (threadData->GetLoad() >= coreLoad / 2) return false; - CoreEntry* other = gCoreLoadHeap->PeekMaximum(); + CoreEntry* other = gCoreLoadHeap.PeekMaximum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); + other = gCoreHighLoadHeap.PeekMinimum(); ASSERT(other != NULL); - return coreLoad - get_core_load(other) >= kLoadDifference / 2; + return coreLoad - other->GetLoad() >= kLoadDifference / 2; } if (coreLoad >= kMediumLoad) return false; - int32 smallTaskCore = choose_small_task_core(); - if (smallTaskCore == -1) + CoreEntry* smallTaskCore = choose_small_task_core(); + if (smallTaskCore == NULL) return false; return smallTaskCore != core - && get_core_load(&gCoreEntries[smallTaskCore]) - + thread->scheduler_data->load < kHighLoad; + && smallTaskCore->GetLoad() +threadData->GetLoad() < kHighLoad; } static inline void pack_irqs(void) { + CoreEntry* smallTaskCore = atomic_pointer_get(&sSmallTaskCore); + if (smallTaskCore == NULL) + return; + cpu_ent* cpu = get_cpu_struct(); - int32 core = gCPUToCore[cpu->cpu_num]; + if (smallTaskCore == CoreEntry::GetCore(cpu->cpu_num)) + return; SpinLocker locker(cpu->irqs_lock); - while (sSmallTaskCore != core && list_get_first_item(&cpu->irqs) != NULL) { + while (list_get_first_item(&cpu->irqs) != NULL) { irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); ReadSpinLocker coreLocker(gCoreHeapsLock); - int32 newCPU - = gCPUPriorityHeaps[sSmallTaskCore].PeekMinimum()->fCPUNumber; + int32 newCPU = smallTaskCore->fCPUHeap.PeekMinimum()->fCPUNumber; coreLocker.Unlock(); if (newCPU != cpu->cpu_num) @@ -193,12 +191,12 @@ pack_irqs(void) static void rebalance_irqs(bool idle) { - if (idle && sSmallTaskCore != -1) { + if (idle && sSmallTaskCore != NULL) { pack_irqs(); return; } - if (idle || sSmallTaskCore != -1) + if (idle || sSmallTaskCore != NULL) return; cpu_ent* cpu = get_cpu_struct(); @@ -219,22 +217,19 @@ rebalance_irqs(bool idle) return; ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); coreLocker.Unlock(); if (other == NULL) return; SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; + int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; cpuLocker.Unlock(); - int32 thisCore = gCPUToCore[smp_get_current_cpu()]; - if (other->fCoreID == thisCore) + CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); + if (other == core) return; - - if (get_core_load(other) + kLoadDifference - >= get_core_load(&gCoreEntries[thisCore])) { + if (other->GetLoad() + kLoadDifference >= core->GetLoad()) return; - } assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 6f71982e58..18c878470e 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -29,13 +29,90 @@ #include #include -#include - #include "scheduler_common.h" +#include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_thread.h" #include "scheduler_tracing.h" +namespace Scheduler { + + +class SchedulerModeLocker : public ReadSpinLocker { +public: + SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) + : + ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, + alreadyLocked, lockIfNotLocked) + { + } +}; + +class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { +public: + InterruptsSchedulerModeLocker(bool alreadyLocked = false, + bool lockIfNotLocked = true) + : + InterruptsReadSpinLocker( + gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, + alreadyLocked, lockIfNotLocked) + { + } +}; + +class BigSchedulerLocking { +public: + bool Lock(int* lockable) + { + *lockable = disable_interrupts(); + for (int32 i = 0; i < smp_get_num_cpus(); i++) + acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + return true; + } + + void Unlock(int* lockable) + { + for (int32 i = 0; i < smp_get_num_cpus(); i++) + release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + restore_interrupts(*lockable); + } +}; + +class InterruptsBigSchedulerLocker : + public AutoLocker { +public: + InterruptsBigSchedulerLocker() + : + AutoLocker(&fState, false, true) + { + } + +private: + int fState; +}; + +scheduler_mode gCurrentModeID; +scheduler_mode_operations* gCurrentMode; + +bool gSingleCore; + +CPUEntry* gCPUEntries; + +CoreEntry* gCoreEntries; +CoreLoadHeap gCoreLoadHeap; +CoreLoadHeap gCoreHighLoadHeap; +rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; +int32 gCoreCount; + +PackageEntry* gPackageEntries; +IdlePackageList gIdlePackageList; +rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; +int32 gPackageCount; + + +} // namespace Scheduler + using namespace Scheduler; @@ -44,581 +121,22 @@ static bool sSchedulerEnabled; SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -static scheduler_mode sCurrentModeID; -static scheduler_mode_operations* sCurrentMode; static scheduler_mode_operations* sSchedulerModes[] = { &gSchedulerLowLatencyMode, &gSchedulerPowerSavingMode, }; -namespace Scheduler { - -bool gSingleCore; - -CPUEntry* gCPUEntries; -CPUHeap* gCPUPriorityHeaps; - -CoreEntry* gCoreEntries; -CoreLoadHeap* gCoreLoadHeap; -CoreLoadHeap* gCoreHighLoadHeap; -rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; -int32 gCoreCount; - -PackageEntry* gPackageEntries; -IdlePackageList* gIdlePackageList; -rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; -int32 gPackageCount; - -ThreadRunQueue* gRunQueues; -ThreadRunQueue* gPinnedRunQueues; - -int32* gCPUToCore; -int32* gCPUToPackage; - -class SchedulerModeLocker : public ReadSpinLocker { -public: - inline SchedulerModeLocker(bool alreadyLocked = false, - bool lockIfNotLocked = true) - : - ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) - { - } -}; - -class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { -public: - inline InterruptsSchedulerModeLocker(bool alreadyLocked = false, - bool lockIfNotLocked = true) - : - InterruptsReadSpinLocker( - gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) - { - } -}; - -} // namespace Scheduler - -static CPUHeap* sDebugCPUHeap; -static CoreLoadHeap* sDebugCoreHeap; - - -CPUEntry::CPUEntry() - : - fPriority(B_IDLE_PRIORITY), - fMeasureActiveTime(0), - fMeasureTime(0), - fLoad(0) -{ - B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); -} - - -CoreEntry::CoreEntry() - : - fCPUCount(0), - fStarvationCounter(0), - fThreadCount(0), - fActiveTime(0), - fLoad(0), - fHighLoad(false) -{ - B_INITIALIZE_SPINLOCK(&fCPULock); - B_INITIALIZE_SPINLOCK(&fQueueLock); -} - - -PackageEntry::PackageEntry() - : - fIdleCoreCount(0), - fCoreCount(0) -{ - B_INITIALIZE_RW_SPINLOCK(&fCoreLock); -} - - -scheduler_thread_data::scheduler_thread_data() -{ - Init(); -} - - -void -scheduler_thread_data::Init() -{ - priority_penalty = 0; - additional_penalty = 0; - - time_left = 0; - stolen_time = 0; - - measure_active_time = 0; - measure_time = 0; - load = 0; - - went_sleep = 0; - went_sleep_active = 0; - went_sleep_count = -1; - - previous_core = -1; - enqueued = false; -} - - -static void -dump_queue(ThreadRunQueue::ConstIterator& iterator) -{ - if (!iterator.HasNext()) - kprintf("Run queue is empty.\n"); - else { - kprintf("thread id priority penalty name\n"); - while (iterator.HasNext()) { - Thread* thread = iterator.Next(); - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", - thread, thread->id, thread->priority, - get_thread_penalty(thread), thread->name); - } - } -} - - -static int -dump_run_queue(int argc, char **argv) -{ - int32 cpuCount = smp_get_num_cpus(); - int32 coreCount = gCoreCount; - - ThreadRunQueue::ConstIterator iterator; - for (int32 i = 0; i < coreCount; i++) { - kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); - iterator = gRunQueues[i].GetConstIterator(); - dump_queue(iterator); - } - - for (int32 i = 0; i < cpuCount; i++) { - iterator = gPinnedRunQueues[i].GetConstIterator(); - - if (iterator.HasNext() && !thread_is_idle_thread(iterator.Next())) { - kprintf("\nCPU %" B_PRId32 " run queue:\n", i); - dump_queue(iterator); - } - } - - return 0; -} - - -static void -dump_cpu_load_heap(CPUHeap* heap) -{ - kprintf("cpu priority load\n"); - CPUEntry* entry = heap->PeekMinimum(); - while (entry) { - int32 cpu = entry->fCPUNumber; - int32 key = CPUHeap::GetKey(entry); - kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, - gCPUEntries[cpu].fLoad / 10); - - heap->RemoveMinimum(); - sDebugCPUHeap->Insert(entry, key); - - entry = heap->PeekMinimum(); - } - - entry = sDebugCPUHeap->PeekMinimum(); - while (entry) { - int32 key = CPUHeap::GetKey(entry); - sDebugCPUHeap->RemoveMinimum(); - heap->Insert(entry, key); - entry = sDebugCPUHeap->PeekMinimum(); - } -} - - -static void -dump_core_load_heap(CoreLoadHeap* heap) -{ - CoreEntry* entry = heap->PeekMinimum(); - while (entry) { - int32 key = CoreLoadHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, - get_core_load(entry) / 10); - - heap->RemoveMinimum(); - sDebugCoreHeap->Insert(entry, key); - - entry = heap->PeekMinimum(); - } - - entry = sDebugCoreHeap->PeekMinimum(); - while (entry) { - int32 key = CoreLoadHeap::GetKey(entry); - sDebugCoreHeap->RemoveMinimum(); - heap->Insert(entry, key); - entry = sDebugCoreHeap->PeekMinimum(); - } -} - - -static int -dump_cpu_heap(int argc, char** argv) -{ - kprintf("core load\n"); - dump_core_load_heap(gCoreLoadHeap); - kprintf("\n"); - dump_core_load_heap(gCoreHighLoadHeap); - - for (int32 i = 0; i < gCoreCount; i++) { - if (gCoreEntries[i].fCPUCount < 2) - continue; - - kprintf("\nCore %" B_PRId32 " heap:\n", i); - dump_cpu_load_heap(&gCPUPriorityHeaps[i]); - } - - return 0; -} - - -static int -dump_idle_cores(int argc, char** argv) -{ - kprintf("Idle packages:\n"); - IdlePackageList::ReverseIterator idleIterator - = gIdlePackageList->GetReverseIterator(); - - if (idleIterator.HasNext()) { - kprintf("package cores\n"); - - while (idleIterator.HasNext()) { - PackageEntry* entry = idleIterator.Next(); - kprintf("%-7" B_PRId32 " ", entry->fPackageID); - - DoublyLinkedList::ReverseIterator iterator - = entry->fIdleCores.GetReverseIterator(); - if (iterator.HasNext()) { - while (iterator.HasNext()) { - CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, - iterator.HasNext() ? ", " : ""); - } - } else - kprintf("-"); - kprintf("\n"); - } - } else - kprintf("No idle packages.\n"); - - return 0; -} - - -static inline bool -has_cache_expired(Thread* thread) -{ - return sCurrentMode->has_cache_expired(thread); -} +// Since CPU IDs used internally by the kernel bear no relation to the actual +// CPU topology the following arrays are used to efficiently get the core +// and the package that CPU in question belongs to. +static int32* sCPUToCore; +static int32* sCPUToPackage; void scheduler_dump_thread_data(Thread* thread) { - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", - schedulerThreadData->priority_penalty); - - int32 additionalPenalty = 0; - const int kMinimalPriority = get_minimal_priority(thread); - if (kMinimalPriority > 0) { - additionalPenalty - = schedulerThreadData->additional_penalty % kMinimalPriority; - } - kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", - additionalPenalty, schedulerThreadData->additional_penalty); - kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", - schedulerThreadData->stolen_time); - kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", schedulerThreadData->load / 10); - kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", - schedulerThreadData->went_sleep); - kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", - schedulerThreadData->went_sleep_active); - kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", - schedulerThreadData->went_sleep_count); - kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", - schedulerThreadData->previous_core); - if (schedulerThreadData->previous_core > 0 - && has_cache_expired(thread)) { - kprintf("\tcache affinity has expired\n"); - } -} - - -static void -update_load_heaps(int32 core) -{ - ASSERT(!gSingleCore); - - CoreEntry* entry = &gCoreEntries[core]; - - if (entry->fCPUCount == 0) { - entry->fLoad = 0; - return; - } - - WriteSpinLocker coreLocker(gCoreHeapsLock); - - int32 newKey = get_core_load(entry); - int32 oldKey = CoreLoadHeap::GetKey(entry); - - ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); - ASSERT(newKey >= 0 && newKey <= kMaxLoad); - - if (oldKey == newKey) - return; - - if (newKey > kHighLoad) { - if (!entry->fHighLoad) { - gCoreLoadHeap->ModifyKey(entry, -1); - ASSERT(gCoreLoadHeap->PeekMinimum() == entry); - gCoreLoadHeap->RemoveMinimum(); - - gCoreHighLoadHeap->Insert(entry, newKey); - - entry->fHighLoad = true; - } else - gCoreHighLoadHeap->ModifyKey(entry, newKey); - } else if (newKey < kMediumLoad) { - if (entry->fHighLoad) { - gCoreHighLoadHeap->ModifyKey(entry, -1); - ASSERT(gCoreHighLoadHeap->PeekMinimum() == entry); - gCoreHighLoadHeap->RemoveMinimum(); - - gCoreLoadHeap->Insert(entry, newKey); - - entry->fHighLoad = false; - } else - gCoreLoadHeap->ModifyKey(entry, newKey); - } else { - if (entry->fHighLoad) - gCoreHighLoadHeap->ModifyKey(entry, newKey); - else - gCoreLoadHeap->ModifyKey(entry, newKey); - } -} - - -static inline void -increase_penalty(Thread* thread) -{ - if (thread->priority < B_LOWEST_ACTIVE_PRIORITY) - return; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return; - - TRACE("increasing thread %ld penalty\n", thread->id); - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - int32 oldPenalty = schedulerThreadData->priority_penalty++; - - ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - - const int kMinimalPriority = get_minimal_priority(thread); - if (thread->priority - oldPenalty <= kMinimalPriority) { - schedulerThreadData->priority_penalty = oldPenalty; - schedulerThreadData->additional_penalty++; - } -} - - -static inline void -cancel_penalty(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (schedulerThreadData->priority_penalty != 0) - TRACE("cancelling thread %ld penalty\n", thread->id); - - schedulerThreadData->additional_penalty = 0; - schedulerThreadData->priority_penalty = 0; -} - - -static inline void -update_cpu_priority(int32 cpu, int32 priority) -{ - int32 core = gCPUToCore[cpu]; - int32 corePriority = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); - gCPUPriorityHeaps[core].ModifyKey(&gCPUEntries[cpu], priority); - - if (gSingleCore) - return; - - int32 maxPriority - = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); - - if (corePriority == maxPriority) - return; - - int32 package = gCPUToPackage[cpu]; - PackageEntry* packageEntry = &gPackageEntries[package]; - if (maxPriority == B_IDLE_PRIORITY) { - WriteSpinLocker _(packageEntry->fCoreLock); - - // core goes idle - ASSERT(packageEntry->fIdleCoreCount >= 0); - ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount++; - packageEntry->fIdleCores.Add(&gCoreEntries[core]); - - if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { - // package goes idle - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList->Add(packageEntry); - } - } else if (corePriority == B_IDLE_PRIORITY) { - WriteSpinLocker _(packageEntry->fCoreLock); - - // core wakes up - ASSERT(packageEntry->fIdleCoreCount > 0); - ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount--; - packageEntry->fIdleCores.Remove(&gCoreEntries[core]); - - if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { - // package wakes up - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList->Remove(packageEntry); - } - } -} - - -static inline int32 -choose_core(Thread* thread) -{ - ASSERT(!gSingleCore); - return sCurrentMode->choose_core(thread); -} - - -static inline int32 -choose_cpu(int32 core, Thread* thread, bool& rescheduleNeeded) -{ - SpinLocker cpuLocker(gCoreEntries[core].fCPULock); - CPUEntry* entry = gCPUPriorityHeaps[core].PeekMinimum(); - ASSERT(entry != NULL); - - int32 threadPriority = get_effective_priority(thread); - if (CPUHeap::GetKey(entry) < threadPriority) { - update_cpu_priority(entry->fCPUNumber, threadPriority); - rescheduleNeeded = true; - } else - rescheduleNeeded = false; - - return entry->fCPUNumber; -} - - -static bool -choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) -{ - bool rescheduleNeeded = false; - - if (targetCore == -1 && targetCPU != -1) - targetCore = gCPUToCore[targetCPU]; - else if (targetCore != -1 && targetCPU == -1) - targetCPU = choose_cpu(targetCore, thread, rescheduleNeeded); - else if (targetCore == -1 && targetCPU == -1) { - targetCore = choose_core(thread); - targetCPU = choose_cpu(targetCore, thread, rescheduleNeeded); - } - - ASSERT(targetCore >= 0 && targetCore < gCoreCount); - ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); - - return rescheduleNeeded; -} - - -static bool -should_rebalance(Thread* thread) -{ - ASSERT(!gSingleCore); - - return sCurrentMode->should_rebalance(thread); -} - - -static inline void -compute_cpu_load(int32 cpu) -{ - ASSERT(!gSingleCore); - - int oldLoad = compute_load(gCPUEntries[cpu].fMeasureTime, - gCPUEntries[cpu].fMeasureActiveTime, gCPUEntries[cpu].fLoad); - if (oldLoad < 0) - return; - - if (oldLoad != gCPUEntries[cpu].fLoad) { - int32 core = gCPUToCore[cpu]; - - int32 delta = gCPUEntries[cpu].fLoad - oldLoad; - atomic_add(&gCoreEntries[core].fLoad, delta); - - update_load_heaps(core); - } - - if (gCPUEntries[cpu].fLoad > kVeryHighLoad) - sCurrentMode->rebalance_irqs(false); -} - - -static inline void -compute_thread_load(Thread* thread) -{ - if (thread->scheduler_data->last_interrupt_time > 0) { - bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; - interruptTime -= thread->scheduler_data->last_interrupt_time; - thread->scheduler_data->measure_active_time -= interruptTime; - } - - compute_load(thread->scheduler_data->measure_time, - thread->scheduler_data->measure_active_time, - thread->scheduler_data->load); -} - - -static inline void -thread_goes_away(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - schedulerThreadData->last_interrupt_time = 0; - - schedulerThreadData->went_sleep = system_time(); - schedulerThreadData->went_sleep_active - = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); - schedulerThreadData->went_sleep_count - = atomic_get(&gCoreEntries[smp_get_current_cpu()].fStarvationCounter); -} - - -static inline bool -should_cancel_penalty(Thread* thread) -{ - int32 core = thread->scheduler_data->previous_core; - - if (core < 0) - return false; - - return atomic_get(&gCoreEntries[core].fStarvationCounter) - != thread->scheduler_data->went_sleep_count - && system_time() - thread->scheduler_data->went_sleep - > sCurrentMode->base_quantum; + thread->scheduler_data->Dump(); } @@ -627,62 +145,45 @@ enqueue(Thread* thread, bool newOne) { ASSERT(thread != NULL); - thread->state = B_THREAD_READY; - - compute_thread_load(thread); - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - schedulerThreadData->time_left = 0; - schedulerThreadData->went_sleep_count = 0; - int32 threadPriority = get_effective_priority(thread); + ThreadData* threadData = thread->scheduler_data; + int32 threadPriority = threadData->GetEffectivePriority(); T(EnqueueThread(thread, threadPriority)); - bool pinned = thread->pinned_to_cpu > 0; - int32 targetCPU = -1; - int32 targetCore = -1; - if (pinned) - targetCPU = thread->previous_cpu->cpu_num; - else if (gSingleCore) - targetCore = 0; - else if (schedulerThreadData->previous_core >= 0 - && (!newOne || !has_cache_expired(thread)) - && !should_rebalance(thread)) { - targetCore = schedulerThreadData->previous_core; + CPUEntry* targetCPU = NULL; + CoreEntry* targetCore = NULL; + if (thread->pinned_to_cpu > 0) { + ASSERT(thread->previous_cpu != NULL); + targetCPU = &gCPUEntries[thread->previous_cpu->cpu_num]; + } else if (gSingleCore) + targetCore = &gCoreEntries[0]; + else if (threadData->GetCore() != NULL + && (!newOne || !threadData->HasCacheExpired()) + && !threadData->ShouldRebalance()) { + targetCore = threadData->GetCore(); } - bool rescheduleNeeded = choose_core_and_cpu(thread, targetCore, targetCPU); - schedulerThreadData->previous_core = targetCore; + bool rescheduleNeeded = threadData->ChooseCoreAndCPU(targetCore, targetCPU); TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", - thread->id, threadPriority, targetCPU, targetCore); + thread->id, threadPriority, targetCPU->fCPUNumber, targetCore->fCoreID); - SpinLocker runQueueLocker(gCoreEntries[targetCore].fQueueLock); - thread->scheduler_data->enqueued = true; - if (pinned) - gPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); - else { - gRunQueues[targetCore].PushBack(thread, threadPriority); - gCoreEntries[targetCore].fThreadList.Insert(thread->scheduler_data); - - atomic_add(&gCoreEntries[targetCore].fThreadCount, 1); - } - runQueueLocker.Unlock(); + threadData->Enqueue(); // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - int32 heapPriority = CPUHeap::GetKey(&gCPUEntries[targetCPU]); - if (threadPriority > atomic_get(&gCPUEntries[targetCPU].fPriority) + int32 heapPriority = CPUPriorityHeap::GetKey(targetCPU); + if (threadPriority > atomic_get(&targetCPU->fPriority) && (threadPriority > heapPriority || (threadPriority == heapPriority && rescheduleNeeded))) { - if (targetCPU == smp_get_current_cpu()) - gCPU[targetCPU].invoke_scheduler = true; + if (targetCPU->fCPUNumber == smp_get_current_cpu()) + gCPU[targetCPU->fCPUNumber].invoke_scheduler = true; else { - smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, - SMP_MSG_FLAG_ASYNC); + smp_send_ici(targetCPU->fCPUNumber, SMP_MSG_RESCHEDULE, 0, 0, 0, + NULL, SMP_MSG_FLAG_ASYNC); } } } @@ -699,43 +200,15 @@ scheduler_enqueue_in_run_queue(Thread *thread) TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ThreadData* threadData = thread->scheduler_data; - int32 core = schedulerThreadData->previous_core; - if (core >= 0) { - if (should_cancel_penalty(thread)) - cancel_penalty(thread); - } + if (threadData->ShouldCancelPenalty()) + threadData->CancelPenalty(); enqueue(thread, true); } -static inline void -put_back(Thread* thread) -{ - compute_thread_load(thread); - thread->scheduler_data->went_sleep_count = -1; - - int32 core = gCPUToCore[smp_get_current_cpu()]; - - SpinLocker runQueueLocker(gCoreEntries[core].fQueueLock); - thread->scheduler_data->enqueued = true; - if (thread->pinned_to_cpu > 0) { - int32 pinnedCPU = thread->previous_cpu->cpu_num; - - ASSERT(pinnedCPU == smp_get_current_cpu()); - gPinnedRunQueues[pinnedCPU].PushFront(thread, - get_effective_priority(thread)); - } else { - ASSERT(thread->scheduler_data->previous_core == core); - - gRunQueues[core].PushFront(thread, get_effective_priority(thread)); - atomic_add(&gCoreEntries[core].fThreadCount, 1); - } -} - - /*! Sets the priority of a thread. */ int32 @@ -744,73 +217,44 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) InterruptsSpinLocker _(thread->scheduler_lock); SchedulerModeLocker modeLocker; + ThreadData* threadData = thread->scheduler_data; int32 oldPriority = thread->priority; TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", - thread->id, priority, oldPriority, get_effective_priority(thread)); + thread->id, priority, oldPriority, threadData->GetEffectivePriority()); - cancel_penalty(thread); + threadData->CancelPenalty(); if (priority == thread->priority) return thread->priority; - thread->priority = priority; - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); - if (thread->state != B_THREAD_READY) { if (thread->state == B_THREAD_RUNNING) { - ASSERT(thread->previous_cpu != NULL); + ASSERT(threadData->GetCore() != NULL); - SpinLocker coreLocker(gCoreEntries[previousCore].fCPULock); + ASSERT(thread->cpu != NULL); + CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num]; - gCPUEntries[thread->cpu->cpu_num].fPriority = priority; - update_cpu_priority(thread->cpu->cpu_num, priority); + SpinLocker coreLocker(threadData->GetCore()->fCPULock); + cpu->fPriority = priority; + cpu->UpdatePriority(priority); } + return oldPriority; } // The thread is in the run queue. We need to remove it and re-insert it at // a new position. - bool pinned = thread->pinned_to_cpu > 0; - int32 previousCPU; - if (pinned) { - ASSERT(thread->previous_cpu != NULL); - previousCPU = thread->previous_cpu->cpu_num; - } + T(RemoveThread(thread)); - SpinLocker runQueueLocker(gCoreEntries[previousCore].fQueueLock); - - // the thread might have been already dequeued and is about to start - // running once we release its scheduler_lock, in such case we can not - // attempt to dequeue it - if (thread->scheduler_data->enqueued) { - T(RemoveThread(thread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, - thread); - - thread->scheduler_data->enqueued = false; - if (pinned) - gPinnedRunQueues[previousCPU].Remove(thread); - else { - gRunQueues[previousCore].Remove(thread); - - ASSERT(thread->scheduler_data->went_sleep_count < 1); - if (thread->scheduler_data->went_sleep_count == 0) { - gCoreEntries[previousCore].fThreadList.Remove( - thread->scheduler_data); - } - - atomic_add(&gCoreEntries[previousCore].fThreadCount, -1); - } - runQueueLocker.Unlock(); + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, + thread); + if (threadData->Dequeue()) enqueue(thread, true); - } return oldPriority; } @@ -841,212 +285,6 @@ reschedule_event(timer* /* unused */) } -static inline bool -quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (hasYielded) { - schedulerThreadData->time_left = 0; - return true; - } - - bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; - schedulerThreadData->time_left -= time_used; - schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); - - // too little time left, it's better make the next quantum a bit longer - if (wasPreempted - || schedulerThreadData->time_left <= sCurrentMode->minimal_quantum) { - - schedulerThreadData->stolen_time += schedulerThreadData->time_left; - schedulerThreadData->time_left = 0; - } - - return schedulerThreadData->time_left == 0; -} - - -static inline bigtime_t -quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, - int32 maxPriority, int32 minPriority, int32 priority) -{ - ASSERT(priority <= maxPriority); - ASSERT(priority >= minPriority); - - bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); - result /= maxPriority - minPriority; - return maxQuantum - result; -} - - -static inline bigtime_t -get_base_quantum(Thread* thread) -{ - int32 priority = get_effective_priority(thread); - - const bigtime_t kQuantum0 = sCurrentMode->base_quantum; - if (priority >= B_URGENT_DISPLAY_PRIORITY) - return kQuantum0; - - const bigtime_t kQuantum1 - = kQuantum0 * sCurrentMode->quantum_multipliers[0]; - if (priority > B_NORMAL_PRIORITY) { - return quantum_linear_interpolation(kQuantum1, kQuantum0, - B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, priority); - } - - const bigtime_t kQuantum2 - = kQuantum0 * sCurrentMode->quantum_multipliers[1]; - return quantum_linear_interpolation(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, - B_IDLE_PRIORITY, priority); -} - - -static inline bigtime_t -compute_quantum(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - bigtime_t quantum; - if (schedulerThreadData->time_left != 0) - quantum = schedulerThreadData->time_left; - else - quantum = get_base_quantum(thread); - - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return quantum; - - quantum += schedulerThreadData->stolen_time; - schedulerThreadData->stolen_time = 0; - - ASSERT(schedulerThreadData->previous_core - == gCPUToCore[smp_get_current_cpu()]); - CoreEntry* core = &gCoreEntries[schedulerThreadData->previous_core]; - int32 threadCount = (core->fThreadCount + 1) / core->fCPUCount; - threadCount = max_c(threadCount, 1); - - quantum = max_c(min_c(sCurrentMode->maximum_latency / threadCount, quantum), - sCurrentMode->minimal_quantum); - - schedulerThreadData->time_left = quantum; - schedulerThreadData->quantum_start = system_time(); - return quantum; -} - - -static inline Thread* -choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) -{ - int32 thisCore = gCPUToCore[thisCPU]; - - SpinLocker runQueueLocker(gCoreEntries[thisCore].fQueueLock); - - Thread* sharedThread = gRunQueues[thisCore].PeekMaximum(); - Thread* pinnedThread = gPinnedRunQueues[thisCPU].PeekMaximum(); - - ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); - - int32 pinnedPriority = -1; - if (pinnedThread != NULL) - pinnedPriority = get_effective_priority(pinnedThread); - - int32 sharedPriority = -1; - if (sharedThread != NULL) - sharedPriority = get_effective_priority(sharedThread); - - int32 oldPriority = -1; - if (oldThread != NULL) - oldPriority = get_effective_priority(oldThread); - - int32 rest = max_c(pinnedPriority, sharedPriority); - if (oldPriority > rest || (!putAtBack && oldPriority == rest)) { - ASSERT(!oldThread->scheduler_data->enqueued); - return oldThread; - } - - if (sharedPriority > pinnedPriority) { - ASSERT(sharedThread->scheduler_data->enqueued); - sharedThread->scheduler_data->enqueued = false; - - gRunQueues[thisCore].Remove(sharedThread); - if (thread_is_idle_thread(sharedThread) - || gCoreEntries[thisCore].fThreadList.Head() - == sharedThread->scheduler_data) { - atomic_add(&gCoreEntries[thisCore].fStarvationCounter, 1); - } - - if (sharedThread->scheduler_data->went_sleep_count == 0) { - gCoreEntries[thisCore].fThreadList.Remove( - sharedThread->scheduler_data); - } - - atomic_add(&gCoreEntries[thisCore].fThreadCount, -1); - return sharedThread; - } - - ASSERT(pinnedThread->scheduler_data->enqueued); - pinnedThread->scheduler_data->enqueued = false; - - gPinnedRunQueues[thisCPU].Remove(pinnedThread); - return pinnedThread; -} - - -static inline void -track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) -{ - if (!thread_is_idle_thread(oldThread)) { - bigtime_t active - = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time); - - atomic_add64(&oldThread->cpu->active_time, active); - oldThread->scheduler_data->measure_active_time += active; - - gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - atomic_add64(&gCoreEntries[thisCore].fActiveTime, active); - } - - compute_thread_load(oldThread); - compute_thread_load(nextThread); - if (!gSingleCore && !gCPU[smp_get_current_cpu()].disabled) - compute_cpu_load(smp_get_current_cpu()); - - if (!thread_is_idle_thread(nextThread)) { - oldThread->cpu->last_kernel_time = nextThread->kernel_time; - oldThread->cpu->last_user_time = nextThread->user_time; - } -} - - -static inline void -update_cpu_performance(Thread* thread, int32 thisCore) -{ - int32 load = max_c(thread->scheduler_data->load, - get_core_load(&gCoreEntries[thisCore])); - load = min_c(max_c(load, 0), kMaxLoad); - - if (load < kTargetLoad) { - int32 delta = kTargetLoad - load; - - delta *= kTargetLoad; - delta /= kCPUPerformanceScaleMax; - - decrease_cpu_performance(delta); - } else { - bool allowBoost = !sCurrentMode->avoid_boost; - allowBoost = allowBoost || thread->scheduler_data->priority_penalty > 0; - - int32 delta = load - kTargetLoad; - delta *= kMaxLoad - kTargetLoad; - delta /= kCPUPerformanceScaleMax; - - increase_cpu_performance(delta, allowBoost); - } -} - - static inline void stop_cpu_timers(Thread* fromThread, Thread* toThread) { @@ -1170,18 +408,19 @@ reschedule(int32 nextState) Thread* oldThread = thread_get_current_thread(); int32 thisCPU = smp_get_current_cpu(); - int32 thisCore = gCPUToCore[thisCPU]; + + CPUEntry* cpu = &gCPUEntries[thisCPU]; + CoreEntry* core = CoreEntry::GetCore(thisCPU); TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, oldThread->id); oldThread->state = nextState; - scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; + ThreadData* oldThreadData = oldThread->scheduler_data; // return time spent in interrupts - schedulerOldThreadData->stolen_time - += gCPU[thisCPU].interrupt_time - - schedulerOldThreadData->last_interrupt_time; + oldThreadData->fStolenTime + += gCPU[thisCPU].interrupt_time - oldThreadData->fLastInterruptTime; bool enqueueOldThread = false; bool putOldThreadAtBack = false; @@ -1190,16 +429,16 @@ reschedule(int32 nextState) case B_THREAD_READY: enqueueOldThread = true; - if (quantum_ended(oldThread, oldThread->cpu->preempted, + if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, oldThread->has_yielded)) { - increase_penalty(oldThread); + oldThreadData->IncreasePenalty(); TRACE("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, get_effective_priority(oldThread)); + oldThread->id, oldThreadData->GetEffectivePriority()); putOldThreadAtBack = true; } else { TRACE("putting thread %ld back in run queue priority = %ld\n", - oldThread->id, get_effective_priority(oldThread)); + oldThread->id, oldThreadData->GetEffectivePriority()); putOldThreadAtBack = false; } @@ -1207,8 +446,8 @@ reschedule(int32 nextState) case THREAD_STATE_FREE_ON_RESCHED: break; default: - increase_penalty(oldThread); - thread_goes_away(oldThread); + oldThreadData->IncreasePenalty(); + oldThreadData->GoesAway(); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, nextState); break; @@ -1217,33 +456,33 @@ reschedule(int32 nextState) oldThread->has_yielded = false; // select thread with the biggest priority and enqueue back the old thread - Thread* nextThread; + ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { if (!thread_is_idle_thread(oldThread)) { - SpinLocker runQueueLocker(gCoreEntries[thisCore].fQueueLock); + SpinLocker runQueueLocker(core->fQueueLock); - nextThread = gPinnedRunQueues[thisCPU].GetHead(B_IDLE_PRIORITY); - gPinnedRunQueues[thisCPU].Remove(nextThread); - nextThread->scheduler_data->enqueued = false; + nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY); + cpu->fRunQueue.Remove(nextThreadData); + nextThreadData->fEnqueued = false; putOldThreadAtBack = oldThread->pinned_to_cpu == 0; } else - nextThread = oldThread; + nextThreadData = oldThreadData; } else { - nextThread - = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, + nextThreadData + = cpu->ChooseNextThread(enqueueOldThread ? oldThreadData : NULL, putOldThreadAtBack); } - atomic_set(&gCPUEntries[thisCPU].fPriority, - get_effective_priority(nextThread)); + Thread* nextThread = nextThreadData->GetThread(); + atomic_set(&cpu->fPriority, nextThreadData->GetEffectivePriority()); if (nextThread != oldThread) { if (enqueueOldThread) { if (putOldThreadAtBack) enqueue(oldThread, false); else - put_back(oldThread); + oldThreadData->PutBack(); } acquire_spinlock(&nextThread->scheduler_lock); @@ -1258,28 +497,20 @@ reschedule(int32 nextState) NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, oldThread, nextThread); - // update CPU heap - if (!gCPU[thisCPU].disabled) { - SpinLocker coreLocker(gCoreEntries[thisCore].fCPULock); - update_cpu_priority(thisCPU, get_effective_priority(nextThread)); - } - + ASSERT(nextThreadData->GetCore() == core); nextThread->state = B_THREAD_RUNNING; - ASSERT(nextThread->scheduler_data->previous_core == thisCore); + // update CPU heap + if (!gCPU[thisCPU].disabled) { + SpinLocker coreLocker(core->fCPULock); + cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); + } // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); // track CPU activity - track_cpu_activity(oldThread, nextThread, thisCore); - - // start counting time spent in interrupts - nextThread->scheduler_data->last_interrupt_time - = gCPU[thisCPU].interrupt_time; - - if (!thread_is_idle_thread(nextThread)) - update_cpu_performance(nextThread, thisCore); + cpu->TrackActivity(oldThreadData, nextThreadData); if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; @@ -1288,13 +519,13 @@ reschedule(int32 nextState) oldThread->cpu->preempted = false; if (!thread_is_idle_thread(nextThread)) { - bigtime_t quantum = compute_quantum(nextThread); + bigtime_t quantum = nextThreadData->ComputeQuantum(); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); } else { - nextThread->scheduler_data->quantum_start = system_time(); + nextThreadData->fQuantumStart = system_time(); - sCurrentMode->rebalance_irqs(true); + gCurrentMode->rebalance_irqs(true); } modeLocker.Unlock(); @@ -1324,7 +555,7 @@ scheduler_reschedule(int32 nextState) status_t scheduler_on_thread_create(Thread* thread, bool idleThread) { - thread->scheduler_data = new (std::nothrow)scheduler_thread_data; + thread->scheduler_data = new(std::nothrow) ThreadData(thread); if (thread->scheduler_data == NULL) return B_NO_MEMORY; return B_OK; @@ -1334,17 +565,16 @@ scheduler_on_thread_create(Thread* thread, bool idleThread) void scheduler_on_thread_init(Thread* thread) { - thread->scheduler_data->Init(); - if (thread_is_idle_thread(thread)) { static int32 sIdleThreadsID; - int32 cpu = atomic_add(&sIdleThreadsID, 1); + int32 cpuID = atomic_add(&sIdleThreadsID, 1); - thread->previous_cpu = &gCPU[cpu]; + thread->previous_cpu = &gCPU[cpuID]; thread->pinned_to_cpu = 1; - thread->scheduler_data->previous_core = gCPUToCore[cpu]; - } + thread->scheduler_data->Init(CoreEntry::GetCore(cpuID)); + } else + thread->scheduler_data->Init(); } @@ -1367,22 +597,6 @@ scheduler_start(void) } -static inline void -acquire_big_scheduler_lock(void) -{ - for (int32 i = 0; i < smp_get_num_cpus(); i++) - acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); -} - - -static inline void -release_big_scheduler_lock(void) -{ - for (int32 i = 0; i < smp_get_num_cpus(); i++) - release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); -} - - status_t scheduler_set_operation_mode(scheduler_mode mode) { @@ -1393,14 +607,11 @@ scheduler_set_operation_mode(scheduler_mode mode) dprintf("scheduler: switching to %s mode\n", sSchedulerModes[mode]->name); - InterruptsLocker _; - acquire_big_scheduler_lock(); + InterruptsBigSchedulerLocker _; - sCurrentModeID = mode; - sCurrentMode = sSchedulerModes[mode]; - sCurrentMode->switch_to_mode(); - - release_big_scheduler_lock(); + gCurrentModeID = mode; + gCurrentMode = sSchedulerModes[mode]; + gCurrentMode->switch_to_mode(); return B_OK; } @@ -1409,38 +620,38 @@ scheduler_set_operation_mode(scheduler_mode mode) static void unassign_thread(Thread* thread, void* data) { - int32 core = *(int32*)data; + CoreEntry* core = static_cast(data); - if (thread->scheduler_data->previous_core == core + if (thread->scheduler_data->GetCore() == core && thread->pinned_to_cpu == 0) { - thread->scheduler_data->previous_core = -1; + thread->scheduler_data->UnassignCore(); } } void -scheduler_set_cpu_enabled(int32 cpu, bool enabled) +scheduler_set_cpu_enabled(int32 cpuID, bool enabled) { dprintf("scheduler: %s CPU %" B_PRId32 "\n", - enabled ? "enabling" : "disabling", cpu); + enabled ? "enabling" : "disabling", cpuID); - InterruptsLocker _; - acquire_big_scheduler_lock(); + InterruptsBigSchedulerLocker _; - gCPU[cpu].disabled = !enabled; + gCPU[cpuID].disabled = !enabled; - sCurrentMode->set_cpu_enabled(cpu, enabled); + gCurrentMode->set_cpu_enabled(cpuID, enabled); - CoreEntry* core = &gCoreEntries[gCPUToCore[cpu]]; - PackageEntry* package = &gPackageEntries[gCPUToPackage[cpu]]; + CPUEntry* cpu = &gCPUEntries[cpuID]; + CoreEntry* core = cpu->fCore; + PackageEntry* package = core->fPackage; int32 oldCPUCount = core->fCPUCount; ASSERT(oldCPUCount >= 0); if (enabled) core->fCPUCount++; else { - gCPUEntries[cpu].fPriority = B_IDLE_PRIORITY; - update_cpu_priority(cpu, B_IDLE_PRIORITY); + cpu->fPriority = B_IDLE_PRIORITY; + cpu->UpdatePriority(B_IDLE_PRIORITY); core->fCPUCount--; } @@ -1449,13 +660,13 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) ASSERT(!enabled); if (core->fHighLoad) { - gCoreHighLoadHeap->ModifyKey(core, -1); - ASSERT(gCoreHighLoadHeap->PeekMinimum() == core); - gCoreHighLoadHeap->RemoveMinimum(); + gCoreHighLoadHeap.ModifyKey(core, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == core); + gCoreHighLoadHeap.RemoveMinimum(); } else { - gCoreLoadHeap->ModifyKey(core, -1); - ASSERT(gCoreLoadHeap->PeekMinimum() == core); - gCoreLoadHeap->RemoveMinimum(); + gCoreLoadHeap.ModifyKey(core, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == core); + gCoreLoadHeap.RemoveMinimum(); } package->fIdleCores.Remove(core); @@ -1463,63 +674,58 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) package->fCoreCount--; if (package->fCoreCount == 0) - gIdlePackageList->Remove(package); + gIdlePackageList.Remove(package); // get rid of threads thread_map(unassign_thread, &core->fCoreID); core->fThreadCount = 0; - while (gRunQueues[core->fCoreID].PeekMaximum() != NULL) { - Thread* thread = gRunQueues[core->fCoreID].PeekMaximum(); - gRunQueues[core->fCoreID].Remove(thread); - thread->scheduler_data->enqueued = false; + while (core->fRunQueue.PeekMaximum() != NULL) { + ThreadData* threadData = core->fRunQueue.PeekMaximum(); - if (thread->scheduler_data->went_sleep_count == 0) { - gCoreEntries[core->fCoreID].fThreadList.Remove( - thread->scheduler_data); - thread->scheduler_data->went_sleep_count = -1;; + core->fRunQueue.Remove(threadData); + threadData->fEnqueued = false; + + if (threadData->fWentSleepCount == 0) { + core->fThreadList.Remove(threadData); + threadData->fWentSleepCount = -1; } - ASSERT(thread->scheduler_data->previous_core == -1); - enqueue(thread, false); + ASSERT(threadData->GetCore() == NULL); + enqueue(threadData->GetThread(), false); } } else if (oldCPUCount == 0) { // core has been reenabled ASSERT(enabled); - gCPUEntries[cpu].fLoad = 0; + cpu->fLoad = 0; core->fLoad = 0; core->fHighLoad = false; - gCoreLoadHeap->Insert(core, 0); + gCoreLoadHeap.Insert(core, 0); package->fCoreCount++; package->fIdleCoreCount++; package->fIdleCores.Add(core); if (package->fCoreCount == 1) - gIdlePackageList->Add(package); + gIdlePackageList.Add(package); } if (enabled) { - gCPUPriorityHeaps[core->fCoreID].Insert(&gCPUEntries[cpu], - B_IDLE_PRIORITY); - gCPUEntries[cpu].fLoad = 0; + core->fCPUHeap.Insert(cpu, B_IDLE_PRIORITY); + cpu->fLoad = 0; } else { - gCPUPriorityHeaps[core->fCoreID].ModifyKey(&gCPUEntries[cpu], - THREAD_MAX_SET_PRIORITY + 1); - ASSERT(gCPUPriorityHeaps[core->fCoreID].PeekMaximum() - == &gCPUEntries[cpu]); - gCPUPriorityHeaps[core->fCoreID].RemoveMaximum(); + core->fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); + ASSERT(core->fCPUHeap.PeekMaximum() == cpu); + core->fCPUHeap.RemoveMaximum(); - ASSERT(gCPUEntries[cpu].fLoad >= 0 - && gCPUEntries[cpu].fLoad <= kMaxLoad); - - core->fLoad -= gCPUEntries[cpu].fLoad; + ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); + core->fLoad -= cpu->fLoad; ASSERT(core->fLoad >= 0); } if (!enabled) { - cpu_ent* entry = &gCPU[cpu]; + cpu_ent* entry = &gCPU[cpuID]; // get rid of irqs SpinLocker locker(entry->irqs_lock); @@ -1536,13 +742,11 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) locker.Unlock(); // don't wait until the thread quantum ends - if (smp_get_current_cpu() != cpu) { - smp_send_ici(cpu, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, + if (smp_get_current_cpu() != cpuID) { + smp_send_ici(cpuID, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); } } - - release_big_scheduler_lock(); } @@ -1551,8 +755,8 @@ traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) { switch (node->level) { case CPU_TOPOLOGY_SMT: - gCPUToCore[node->id] = coreID; - gCPUToPackage[node->id] = packageID; + sCPUToCore[node->id] = coreID; + sCPUToPackage[node->id] = packageID; return; case CPU_TOPOLOGY_CORE: @@ -1577,15 +781,15 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) { cpuCount = smp_get_num_cpus(); - gCPUToCore = new(std::nothrow) int32[cpuCount]; - if (gCPUToCore == NULL) + sCPUToCore = new(std::nothrow) int32[cpuCount]; + if (sCPUToCore == NULL) return B_NO_MEMORY; - ArrayDeleter cpuToCoreDeleter(gCPUToCore); + ArrayDeleter cpuToCoreDeleter(sCPUToCore); - gCPUToPackage = new(std::nothrow) int32[cpuCount]; - if (gCPUToPackage == NULL) + sCPUToPackage = new(std::nothrow) int32[cpuCount]; + if (sCPUToPackage == NULL) return B_NO_MEMORY; - ArrayDeleter cpuToPackageDeleter(gCPUToPackage); + ArrayDeleter cpuToPackageDeleter(sCPUToPackage); coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { @@ -1610,23 +814,6 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) } -static status_t -create_debug_heaps() -{ - sDebugCPUHeap = new(std::nothrow) CPUHeap(smp_get_num_cpus()); - if (sDebugCPUHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter cpuDeleter(sDebugCPUHeap); - - sDebugCoreHeap = new(std::nothrow) CoreLoadHeap(smp_get_num_cpus()); - if (sDebugCoreHeap == NULL) - return B_NO_MEMORY; - - cpuDeleter.Detach(); - return B_OK; -} - - static status_t init() { @@ -1636,120 +823,68 @@ init() packageCount); if (result != B_OK) return result; + gCoreCount = coreCount; gSingleCore = coreCount == 1; gPackageCount = packageCount; - // create package heap and idle package stack + gCPUEntries = new(std::nothrow) CPUEntry[cpuCount]; + if (gCPUEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuEntriesDeleter(gCPUEntries); + + gCoreEntries = new(std::nothrow) CoreEntry[coreCount]; + if (gCoreEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter coreEntriesDeleter(gCoreEntries); + gPackageEntries = new(std::nothrow) PackageEntry[packageCount]; if (gPackageEntries == NULL) return B_NO_MEMORY; ArrayDeleter packageEntriesDeleter(gPackageEntries); - gIdlePackageList = new(std::nothrow) IdlePackageList; - if (gIdlePackageList == NULL) - return B_NO_MEMORY; - ObjectDeleter packageListDeleter(gIdlePackageList); + new(&gCoreLoadHeap) CoreLoadHeap(coreCount); + new(&gCoreHighLoadHeap) CoreLoadHeap(coreCount); + + new(&gIdlePackageList) IdlePackageList; for (int32 i = 0; i < packageCount; i++) { gPackageEntries[i].fPackageID = i; - gPackageEntries[i].fIdleCoreCount = coreCount / packageCount; gPackageEntries[i].fCoreCount = coreCount / packageCount; - gIdlePackageList->Insert(&gPackageEntries[i]); + gIdlePackageList.Insert(&gPackageEntries[i]); } - // create logical processor and core heaps - gCPUEntries = new CPUEntry[cpuCount]; - if (gCPUEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuEntriesDeleter(gCPUEntries); - - gCoreEntries = new CoreEntry[coreCount]; - if (gCoreEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter coreEntriesDeleter(gCoreEntries); - - gCoreLoadHeap = new CoreLoadHeap; - if (gCoreLoadHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter coreLoadHeapDeleter(gCoreLoadHeap); - - gCoreHighLoadHeap = new CoreLoadHeap(coreCount); - if (gCoreHighLoadHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter coreHighLoadHeapDeleter(gCoreHighLoadHeap); - for (int32 i = 0; i < coreCount; i++) { gCoreEntries[i].fCoreID = i; gCoreEntries[i].fCPUCount = cpuCount / coreCount; - result = gCoreLoadHeap->Insert(&gCoreEntries[i], 0); + result = gCoreLoadHeap.Insert(&gCoreEntries[i], 0); if (result != B_OK) return result; } - gCPUPriorityHeaps = new CPUHeap[coreCount]; - if (gCPUPriorityHeaps == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuPriorityHeapDeleter(gCPUPriorityHeaps); - for (int32 i = 0; i < cpuCount; i++) { + CoreEntry* core = &gCoreEntries[sCPUToCore[i]]; + PackageEntry* package = &gPackageEntries[sCPUToPackage[i]]; + gCPUEntries[i].fCPUNumber = i; + gCPUEntries[i].fCore = core; + core->fPackage = package; - int32 core = gCPUToCore[i]; + if (core->fCPUHeap.PeekMaximum() == NULL) { + package->fIdleCoreCount++; + package->fIdleCores.Insert(core); + } - int32 package = gCPUToPackage[i]; - if (gCPUPriorityHeaps[core].PeekMaximum() == NULL) - gPackageEntries[package].fIdleCores.Insert(&gCoreEntries[core]); - - result - = gCPUPriorityHeaps[core].Insert(&gCPUEntries[i], B_IDLE_PRIORITY); + result = core->fCPUHeap.Insert(&gCPUEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; } - // create per-logical processor run queues for pinned threads - TRACE("scheduler_init(): creating %" B_PRId32 " per-cpu queue%s\n", - cpuCount, cpuCount != 1 ? "s" : ""); - - gPinnedRunQueues = new(std::nothrow) ThreadRunQueue[cpuCount]; - if (gPinnedRunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter pinnedRunQueuesDeleter(gPinnedRunQueues); - for (int i = 0; i < cpuCount; i++) { - result = gPinnedRunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - - // create per-core run queues - TRACE("scheduler_init(): creating %" B_PRId32 " per-core queue%s\n", - coreCount, coreCount != 1 ? "s" : ""); - - gRunQueues = new(std::nothrow) ThreadRunQueue[coreCount]; - if (gRunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter runQueuesDeleter(gRunQueues); - for (int i = 0; i < coreCount; i++) { - result = gRunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - - // create temporary heaps for debugging commands - result = create_debug_heaps(); - if (result != B_OK) - return result; - - runQueuesDeleter.Detach(); - pinnedRunQueuesDeleter.Detach(); - coreHighLoadHeapDeleter.Detach(); - coreLoadHeapDeleter.Detach(); - cpuPriorityHeapDeleter.Detach(); + packageEntriesDeleter.Detach(); coreEntriesDeleter.Detach(); cpuEntriesDeleter.Detach(); - packageEntriesDeleter.Detach(); - packageListDeleter.Detach(); + return B_OK; } @@ -1768,15 +903,7 @@ scheduler_init(void) scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); - add_debugger_command_etc("run_queue", &dump_run_queue, - "List threads in run queue", "\nLists threads in run queue", 0); - if (!gSingleCore) { - add_debugger_command_etc("cpu_heap", &dump_cpu_heap, - "List CPUs in CPU priority heap", - "\nList CPUs in CPU priority heap", 0); - add_debugger_command_etc("idle_cores", &dump_idle_cores, - "List idle cores", "\nList idle cores", 0); - } + init_debug_commands(); #if SCHEDULER_TRACING add_debugger_command_etc("scheduler", &cmd_scheduler, @@ -1846,22 +973,23 @@ _user_estimate_max_scheduling_latency(thread_id id) } BReference threadReference(thread, true); - int32 core = thread->scheduler_data->previous_core; - if (core == -1) - core = get_random() % gCoreCount; + ThreadData* threadData = thread->scheduler_data; + CoreEntry* core = threadData->GetCore(); + if (core == NULL) + core = &gCoreEntries[get_random() % gCoreCount]; - int32 threadCount = gCoreEntries[core].fThreadCount; - if (gCoreEntries[core].fCPUCount > 0) - threadCount /= gCoreEntries[core].fCPUCount; + int32 threadCount = core->fThreadCount; + if (core->fCPUCount > 0) + threadCount /= core->fCPUCount; - if (get_effective_priority(thread) > 0) { + if (threadData->GetEffectivePriority() > 0) { threadCount -= threadCount * THREAD_MAX_SET_PRIORITY - / get_effective_priority(thread); + / threadData->GetEffectivePriority(); } - return min_c(max_c(threadCount * sCurrentMode->base_quantum, - sCurrentMode->minimal_quantum), - sCurrentMode->maximum_latency); + return std::min(std::max(threadCount * gCurrentMode->base_quantum, + gCurrentMode->minimal_quantum), + gCurrentMode->maximum_latency); } @@ -1875,6 +1003,6 @@ _user_set_scheduler_mode(int32 mode) int32 _user_get_scheduler_mode(void) { - return sCurrentModeID; + return gCurrentModeID; } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 3069fffa5a..75490dd30b 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -7,6 +7,8 @@ #define KERNEL_SCHEDULER_COMMON_H +#include + #include #include #include @@ -29,6 +31,9 @@ namespace Scheduler { +struct CPUEntry; +struct CoreEntry; + const int kLowLoad = kMaxLoad * 20 / 100; const int kTargetLoad = kMaxLoad * 55 / 100; const int kHighLoad = kMaxLoad * 70 / 100; @@ -39,179 +44,12 @@ const int kLoadDifference = kMaxLoad * 20 / 100; extern bool gSingleCore; -// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical -// processors in the heap belong to. Since there are no cache affinity issues -// at this level and the run queue is shared among all logical processors on -// the core the only real concern is to make lower priority threads give way to -// the higher priority threads. -struct CPUEntry : public MinMaxHeapLinkImpl { - CPUEntry(); - int32 fCPUNumber; - - int32 fPriority; - - bigtime_t fMeasureActiveTime; - bigtime_t fMeasureTime; - - int32 fLoad; - - rw_spinlock fSchedulerModeLock; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; - -extern CPUEntry* gCPUEntries; -extern CPUHeap* gCPUPriorityHeaps; - -struct CoreEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { - CoreEntry(); - - int32 fCoreID; - - int32 fCPUCount; - - spinlock fCPULock; - spinlock fQueueLock; - - int32 fStarvationCounter; - - int32 fThreadCount; - DoublyLinkedList fThreadList; - - bigtime_t fActiveTime; - - int32 fLoad; - bool fHighLoad; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap CoreLoadHeap; - -extern CoreEntry* gCoreEntries; -extern CoreLoadHeap* gCoreLoadHeap; -extern CoreLoadHeap* gCoreHighLoadHeap; -extern rw_spinlock gCoreHeapsLock; -extern int32 gCoreCount; - -// gPackageEntries are used to decide which core should be woken up from the -// idle state. When aiming for performance we should use as many packages as -// possible with as little cores active in each package as possible (so that the -// package can enter any boost mode if it has one and the active core have more -// of the shared cache for themselves. If power saving is the main priority we -// should keep active cores on as little packages as possible (so that other -// packages can go to the deep state of sleep). The heap stores only packages -// with at least one core active and one core idle. The packages with all cores -// idle are stored in sPackageIdleList (in LIFO manner). -struct PackageEntry : public DoublyLinkedListLinkImpl { - PackageEntry(); - - int32 fPackageID; - - rw_spinlock fCoreLock; - - DoublyLinkedList fIdleCores; - int32 fIdleCoreCount; - - int32 fCoreCount; -} CACHE_LINE_ALIGN; -typedef DoublyLinkedList IdlePackageList; - -extern PackageEntry* gPackageEntries; -extern IdlePackageList* gIdlePackageList; -extern rw_spinlock gIdlePackageLock; -extern int32 gPackageCount; - -// The run queues. Holds the threads ready to run ordered by priority. -// One queue per schedulable target per core. Additionally, each -// logical processor has its sPinnedRunQueues used for scheduling -// pinned threads. -typedef RunQueue CACHE_LINE_ALIGN - ThreadRunQueue; - -extern ThreadRunQueue* gRunQueues; -extern ThreadRunQueue* gPinnedRunQueues; - -// Since CPU IDs used internally by the kernel bear no relation to the actual -// CPU topology the following arrays are used to efficiently get the core -// and the package that CPU in question belongs to. -extern int32* gCPUToCore; -extern int32* gCPUToPackage; +void init_debug_commands(void); } // namespace Scheduler -struct scheduler_thread_data : - public DoublyLinkedListLinkImpl { - inline scheduler_thread_data(); - void Init(); - - int32 priority_penalty; - int32 additional_penalty; - - bigtime_t time_left; - bigtime_t stolen_time; - bigtime_t quantum_start; - bigtime_t last_interrupt_time; - - bigtime_t measure_active_time; - bigtime_t measure_time; - int32 load; - - bigtime_t went_sleep; - bigtime_t went_sleep_active; - int32 went_sleep_count; - - int32 previous_core; - - bool enqueued; -}; - - -static inline int32 -get_core_load(struct Scheduler::CoreEntry* core) -{ - return core->fLoad / core->fCPUCount; -} - - -static inline int32 -get_minimal_priority(Thread* thread) -{ - return max_c(min_c(thread->priority, 25) / 5, 1); -} - - -static inline int32 -get_thread_penalty(Thread* thread) -{ - int32 penalty = thread->scheduler_data->priority_penalty; - - const int kMinimalPriority = get_minimal_priority(thread); - if (kMinimalPriority > 0) { - penalty - += thread->scheduler_data->additional_penalty % kMinimalPriority; - } - - return penalty; -} - - -static inline int32 -get_effective_priority(Thread* thread) -{ - if (thread->priority == B_IDLE_PRIORITY) - return thread->priority; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return thread->priority; - - int32 effectivePriority = thread->priority; - effectivePriority -= get_thread_penalty(thread); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; -} - - #endif // KERNEL_SCHEDULER_COMMON_H + diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp new file mode 100644 index 0000000000..c78d62a149 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -0,0 +1,466 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + + +#include "scheduler_cpu.h" + +#include + +#include + +#include "scheduler_thread.h" + + +using namespace Scheduler; + + +static CPUPriorityHeap sDebugCPUHeap; +static CoreLoadHeap sDebugCoreHeap; + + +void +ThreadRunQueue::Dump() const +{ + ThreadRunQueue::ConstIterator iterator = GetConstIterator(); + if (!iterator.HasNext()) + kprintf("Run queue is empty.\n"); + else { + kprintf("thread id priority penalty name\n"); + while (iterator.HasNext()) { + ThreadData* threadData = iterator.Next(); + Thread* thread = threadData->GetThread(); + + kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", + thread, thread->id, thread->priority, + threadData->GetEffectivePriority(), thread->name); + } + } +} + + +CPUEntry::CPUEntry() + : + fPriority(B_IDLE_PRIORITY), + fLoad(0), + fMeasureActiveTime(0), + fMeasureTime(0) +{ + B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); +} + + +void +CPUEntry::UpdatePriority(int32 priority) +{ + int32 corePriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); + fCore->fCPUHeap.ModifyKey(this, priority); + + if (gSingleCore) + return; + + int32 maxPriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); + if (corePriority == maxPriority) + return; + + PackageEntry* packageEntry = fCore->fPackage; + if (maxPriority == B_IDLE_PRIORITY) { + WriteSpinLocker _(packageEntry->fCoreLock); + + // core goes idle + ASSERT(packageEntry->fIdleCoreCount >= 0); + ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount++; + packageEntry->fIdleCores.Add(fCore); + + if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { + // package goes idle + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Add(packageEntry); + } + } else if (corePriority == B_IDLE_PRIORITY) { + WriteSpinLocker _(packageEntry->fCoreLock); + + // core wakes up + ASSERT(packageEntry->fIdleCoreCount > 0); + ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount--; + packageEntry->fIdleCores.Remove(fCore); + + if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { + // package wakes up + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Remove(packageEntry); + } + } +} + + +void +CPUEntry::ComputeLoad() +{ + ASSERT(!gSingleCore); + ASSERT(fCPUNumber == smp_get_current_cpu()); + + int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad); + if (oldLoad < 0) + return; + + if (oldLoad != fLoad) { + int32 delta = fLoad - oldLoad; + atomic_add(&fCore->fLoad, delta); + + fCore->UpdateLoad(); + } + + if (fLoad > kVeryHighLoad) + gCurrentMode->rebalance_irqs(false); +} + + +ThreadData* +CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) +{ + SpinLocker runQueueLocker(fCore->fQueueLock); + + ThreadData* sharedThread = fCore->fRunQueue.PeekMaximum(); + ThreadData* pinnedThread = fRunQueue.PeekMaximum(); + + ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); + + int32 pinnedPriority = -1; + if (pinnedThread != NULL) + pinnedPriority = pinnedThread->GetEffectivePriority(); + + int32 sharedPriority = -1; + if (sharedThread != NULL) + sharedPriority = sharedThread->GetEffectivePriority(); + + int32 oldPriority = -1; + if (oldThread != NULL) + oldPriority = oldThread->GetEffectivePriority(); + + int32 rest = std::max(pinnedPriority, sharedPriority); + if (oldPriority > rest || (!putAtBack && oldPriority == rest)) + return oldThread; + + if (sharedPriority > pinnedPriority) { + sharedThread->fEnqueued = false; + + fCore->fRunQueue.Remove(sharedThread); + if (thread_is_idle_thread(sharedThread->GetThread()) + || fCore->fThreadList.Head() == sharedThread) { + atomic_add(&fCore->fStarvationCounter, 1); + } + + if (sharedThread->fWentSleepCount == 0) + fCore->fThreadList.Remove(sharedThread); + + atomic_add(&fCore->fThreadCount, -1); + return sharedThread; + } + + pinnedThread->fEnqueued = false; + fRunQueue.Remove(pinnedThread); + return pinnedThread; +} + + +void +CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) +{ + cpu_ent* cpuEntry = &gCPU[fCPUNumber]; + + Thread* oldThread = oldThreadData->GetThread(); + if (!thread_is_idle_thread(oldThread)) { + bigtime_t active + = (oldThread->kernel_time - cpuEntry->last_kernel_time) + + (oldThread->user_time - cpuEntry->last_user_time); + + atomic_add64(&cpuEntry->active_time, active); + oldThreadData->UpdateActivity(active); + } + + oldThreadData->ComputeLoad(); + nextThreadData->ComputeLoad(); + if (!gSingleCore && !cpuEntry->disabled) + ComputeLoad(); + + Thread* nextThread = nextThreadData->GetThread(); + if (!thread_is_idle_thread(nextThread)) { + cpuEntry->last_kernel_time = nextThread->kernel_time; + cpuEntry->last_user_time = nextThread->user_time; + + nextThreadData->fLastInterruptTime = cpuEntry->interrupt_time; + + _RequestPerformanceLevel(nextThreadData); + } +} + + +inline void +CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) +{ + int32 load = std::max(threadData->GetLoad(), fCore->GetLoad()); + load = std::min(std::max(load, int32(0)), kMaxLoad); + + if (load < kTargetLoad) { + int32 delta = kTargetLoad - load; + + delta *= kTargetLoad; + delta /= kCPUPerformanceScaleMax; + + decrease_cpu_performance(delta); + } else { + bool allowBoost = !gCurrentMode->avoid_boost; + + int32 delta = load - kTargetLoad; + delta *= kMaxLoad - kTargetLoad; + delta /= kCPUPerformanceScaleMax; + + increase_cpu_performance(delta, allowBoost); + } +} + + +CPUPriorityHeap::CPUPriorityHeap(int32 cpuCount) + : + MinMaxHeap(cpuCount) +{ +} + + +void +CPUPriorityHeap::Dump() +{ + kprintf("cpu priority load\n"); + CPUEntry* entry = PeekMinimum(); + while (entry) { + int32 cpu = entry->fCPUNumber; + int32 key = GetKey(entry); + kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, + entry->fLoad / 10); + + RemoveMinimum(); + sDebugCPUHeap.Insert(entry, key); + + entry = PeekMinimum(); + } + + entry = sDebugCPUHeap.PeekMinimum(); + while (entry) { + int32 key = GetKey(entry); + sDebugCPUHeap.RemoveMinimum(); + Insert(entry, key); + entry = sDebugCPUHeap.PeekMinimum(); + } +} + + +CoreEntry::CoreEntry() + : + fCPUCount(0), + fStarvationCounter(0), + fThreadCount(0), + fActiveTime(0), + fLoad(0), + fHighLoad(false) +{ + B_INITIALIZE_SPINLOCK(&fCPULock); + B_INITIALIZE_SPINLOCK(&fQueueLock); +} + + +void +CoreEntry::UpdateLoad() +{ + ASSERT(!gSingleCore); + + if (fCPUCount == 0) { + fLoad = 0; + return; + } + + WriteSpinLocker coreLocker(gCoreHeapsLock); + + int32 newKey = GetLoad(); + int32 oldKey = CoreLoadHeap::GetKey(this); + + ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); + ASSERT(newKey >= 0 && newKey <= kMaxLoad); + + if (oldKey == newKey) + return; + + if (newKey > kHighLoad) { + if (!fHighLoad) { + gCoreLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == this); + gCoreLoadHeap.RemoveMinimum(); + + gCoreHighLoadHeap.Insert(this, newKey); + + fHighLoad = true; + } else + gCoreHighLoadHeap.ModifyKey(this, newKey); + } else if (newKey < kMediumLoad) { + if (fHighLoad) { + gCoreHighLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); + gCoreHighLoadHeap.RemoveMinimum(); + + gCoreLoadHeap.Insert(this, newKey); + + fHighLoad = false; + } else + gCoreLoadHeap.ModifyKey(this, newKey); + } else { + if (fHighLoad) + gCoreHighLoadHeap.ModifyKey(this, newKey); + else + gCoreLoadHeap.ModifyKey(this, newKey); + } +} + + +CoreLoadHeap::CoreLoadHeap(int32 coreCount) + : + MinMaxHeap(coreCount) +{ +} + + +void +CoreLoadHeap::Dump() +{ + CoreEntry* entry = PeekMinimum(); + while (entry) { + int32 key = GetKey(entry); + kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, + entry->GetLoad() / 10); + + RemoveMinimum(); + sDebugCoreHeap.Insert(entry, key); + + entry = PeekMinimum(); + } + + entry = sDebugCoreHeap.PeekMinimum(); + while (entry) { + int32 key = GetKey(entry); + sDebugCoreHeap.RemoveMinimum(); + Insert(entry, key); + entry = sDebugCoreHeap.PeekMinimum(); + } +} + + +PackageEntry::PackageEntry() + : + fIdleCoreCount(0), + fCoreCount(0) +{ + B_INITIALIZE_RW_SPINLOCK(&fCoreLock); +} + + +static int +dump_run_queue(int argc, char **argv) +{ + int32 cpuCount = smp_get_num_cpus(); + int32 coreCount = gCoreCount; + + + for (int32 i = 0; i < coreCount; i++) { + kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); + gCoreEntries[i].fRunQueue.Dump(); + } + + for (int32 i = 0; i < cpuCount; i++) { + CPUEntry* cpu = &gCPUEntries[i]; + ThreadRunQueue::ConstIterator iterator + = cpu->fRunQueue.GetConstIterator(); + + if (iterator.HasNext() + && !thread_is_idle_thread(iterator.Next()->GetThread())) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", i); + cpu->fRunQueue.Dump(); + } + } + + return 0; +} + + +static int +dump_cpu_heap(int argc, char** argv) +{ + kprintf("core load\n"); + gCoreLoadHeap.Dump(); + kprintf("\n"); + gCoreHighLoadHeap.Dump(); + + for (int32 i = 0; i < gCoreCount; i++) { + if (gCoreEntries[i].fCPUCount < 2) + continue; + + kprintf("\nCore %" B_PRId32 " heap:\n", i); + gCoreEntries[i].fCPUHeap.Dump(); + } + + return 0; +} + + +static int +dump_idle_cores(int argc, char** argv) +{ + kprintf("Idle packages:\n"); + IdlePackageList::ReverseIterator idleIterator + = gIdlePackageList.GetReverseIterator(); + + if (idleIterator.HasNext()) { + kprintf("package cores\n"); + + while (idleIterator.HasNext()) { + PackageEntry* entry = idleIterator.Next(); + kprintf("%-7" B_PRId32 " ", entry->fPackageID); + + DoublyLinkedList::ReverseIterator iterator + = entry->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); + } + } else + kprintf("No idle packages.\n"); + + return 0; +} + + +void Scheduler::init_debug_commands(void) +{ + new(&sDebugCPUHeap) CPUPriorityHeap(smp_get_num_cpus()); + new(&sDebugCoreHeap) CoreLoadHeap(smp_get_num_cpus()); + + add_debugger_command_etc("run_queue", &dump_run_queue, + "List threads in run queue", "\nLists threads in run queue", 0); + if (!gSingleCore) { + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", + "\nList CPUs in CPU priority heap", 0); + add_debugger_command_etc("idle_cores", &dump_idle_cores, + "List idle cores", "\nList idle cores", 0); + } +} + diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h new file mode 100644 index 0000000000..14c1a80836 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -0,0 +1,170 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_CPU_H +#define KERNEL_SCHEDULER_CPU_H + + +#include + +#include +#include + +#include + +#include "RunQueue.h" +#include "scheduler_common.h" +#include "scheduler_modes.h" + + +namespace Scheduler { + + +struct ThreadData; + +struct CPUEntry; +struct CoreEntry; +struct PackageEntry; + +// The run queues. Holds the threads ready to run ordered by priority. +// One queue per schedulable target per core. Additionally, each +// logical processor has its sPinnedRunQueues used for scheduling +// pinned threads. +class ThreadRunQueue : public RunQueue { +public: + void Dump() const; +}; + +struct CPUEntry : public MinMaxHeapLinkImpl { + CPUEntry(); + + void UpdatePriority(int32 priority); + + void ComputeLoad(); + + ThreadData* ChooseNextThread(ThreadData* oldThread, + bool putAtBack); + + void TrackActivity(ThreadData* oldThreadData, + ThreadData* nextThreadData); + + int32 fCPUNumber; + CoreEntry* fCore; + + rw_spinlock fSchedulerModeLock; + + int32 fPriority; + ThreadRunQueue fRunQueue; + + int32 fLoad; + + bigtime_t fMeasureActiveTime; + bigtime_t fMeasureTime; + +private: + inline void _RequestPerformanceLevel( + ThreadData* threadData); + +} CACHE_LINE_ALIGN; + +class CPUPriorityHeap : public MinMaxHeap { +public: + CPUPriorityHeap() { } + CPUPriorityHeap(int32 cpuCount); + + void Dump(); +}; + +struct CoreEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { + CoreEntry(); + + inline int32 GetLoad() const; + void UpdateLoad(); + + static inline CoreEntry* GetCore(int32 cpu); + + int32 fCoreID; + PackageEntry* fPackage; + + int32 fCPUCount; + CPUPriorityHeap fCPUHeap; + spinlock fCPULock; + + int32 fStarvationCounter; + DoublyLinkedList fThreadList; + + int32 fThreadCount; + ThreadRunQueue fRunQueue; + spinlock fQueueLock; + + bigtime_t fActiveTime; + + int32 fLoad; + bool fHighLoad; +} CACHE_LINE_ALIGN; + +class CoreLoadHeap : public MinMaxHeap { +public: + CoreLoadHeap() { } + CoreLoadHeap(int32 coreCount); + + void Dump(); +}; + +// gPackageEntries are used to decide which core should be woken up from the +// idle state. When aiming for performance we should use as many packages as +// possible with as little cores active in each package as possible (so that the +// package can enter any boost mode if it has one and the active core have more +// of the shared cache for themselves. If power saving is the main priority we +// should keep active cores on as little packages as possible (so that other +// packages can go to the deep state of sleep). The heap stores only packages +// with at least one core active and one core idle. The packages with all cores +// idle are stored in gPackageIdleList (in LIFO manner). +struct PackageEntry : public DoublyLinkedListLinkImpl { + PackageEntry(); + + int32 fPackageID; + + DoublyLinkedList fIdleCores; + int32 fIdleCoreCount; + int32 fCoreCount; + rw_spinlock fCoreLock; +} CACHE_LINE_ALIGN; +typedef DoublyLinkedList IdlePackageList; + +extern CPUEntry* gCPUEntries; + +extern CoreEntry* gCoreEntries; +extern CoreLoadHeap gCoreLoadHeap; +extern CoreLoadHeap gCoreHighLoadHeap; +extern rw_spinlock gCoreHeapsLock; +extern int32 gCoreCount; + +extern PackageEntry* gPackageEntries; +extern IdlePackageList gIdlePackageList; +extern rw_spinlock gIdlePackageLock; +extern int32 gPackageCount; + + +inline int32 +CoreEntry::GetLoad() const +{ + ASSERT(fCPUCount >= 0); + return fLoad / fCPUCount; +} + + +/* static */ inline CoreEntry* +CoreEntry::GetCore(int32 cpu) +{ + return gCPUEntries[cpu].fCore; +} + + +} // namespace Scheduler + + +#endif // KERNEL_SCHEDULER_CPU_H + diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 2d790a3967..378befc2fa 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -11,26 +11,40 @@ struct scheduler_mode_operations { - const char* name; + const char* name; - bool avoid_boost; + bool avoid_boost; - bigtime_t base_quantum; - bigtime_t minimal_quantum; - bigtime_t quantum_multipliers[2]; + bigtime_t base_quantum; + bigtime_t minimal_quantum; + bigtime_t quantum_multipliers[2]; - bigtime_t maximum_latency; + bigtime_t maximum_latency; - void (*switch_to_mode)(void); - void (*set_cpu_enabled)(int32 cpu, bool enabled); - bool (*has_cache_expired)(Thread* thread); - int32 (*choose_core)(Thread* thread); - bool (*should_rebalance)(Thread* thread); - void (*rebalance_irqs)(bool idle); + void (*switch_to_mode)(void); + void (*set_cpu_enabled)(int32 cpu, bool enabled); + bool (*has_cache_expired)( + const Scheduler::ThreadData* threadData); + Scheduler::CoreEntry* (*choose_core)( + const Scheduler::ThreadData* threadData); + bool (*should_rebalance)( + const Scheduler::ThreadData* threadData); + void (*rebalance_irqs)(bool idle); }; extern struct scheduler_mode_operations gSchedulerLowLatencyMode; extern struct scheduler_mode_operations gSchedulerPowerSavingMode; + +namespace Scheduler { + + +extern scheduler_mode gCurrentModeID; +extern scheduler_mode_operations* gCurrentMode; + + +} + + #endif // KERNEL_SCHEDULER_MODES_H diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp new file mode 100644 index 0000000000..cf53fcda9f --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -0,0 +1,158 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + +#include "scheduler_thread.h" + + +using namespace Scheduler; + + +ThreadData::ThreadData(Thread* thread) + : + fThread(thread) +{ + Init(); +} + + +void +ThreadData::Init() +{ + fPriorityPenalty = 0; + fAdditionalPenalty = 0; + + fTimeLeft = 0; + fStolenTime = 0; + + fMeasureActiveTime = 0; + fMeasureTime = 0; + fLoad = 0; + + fWentSleep = 0; + fWentSleepActive = 0; + fWentSleepCount = -1; + + fEnqueued = false; + + fCore = NULL; +} + + +void +ThreadData::Init(CoreEntry* core) +{ + Init(); + fCore = core; +} + + +void +ThreadData::Dump() const +{ + kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", fPriorityPenalty); + + int32 additionalPenalty = 0; + const int kMinimalPriority = _GetMinimalPriority(); + if (kMinimalPriority > 0) + additionalPenalty = fAdditionalPenalty % kMinimalPriority; + kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", + additionalPenalty, fAdditionalPenalty); + kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", fStolenTime); + kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", fLoad / 10); + kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", fWentSleep); + kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); + kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount); + kprintf("\tcore:\t\t\t%" B_PRId32 "\n", + fCore != NULL ? fCore->fCoreID : -1); + if (fCore != NULL && HasCacheExpired()) + kprintf("\tcache affinity has expired\n"); +} + + +bool +ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) +{ + bool rescheduleNeeded = false; + + if (targetCore == NULL && targetCPU != NULL) + targetCore = targetCPU->fCore; + else if (targetCore != NULL && targetCPU == NULL) + targetCPU = _ChooseCPU(targetCore, rescheduleNeeded); + else if (targetCore == NULL && targetCPU == NULL) { + targetCore = _ChooseCore(); + targetCPU = _ChooseCPU(targetCore, rescheduleNeeded); + } + + ASSERT(targetCore != NULL); + ASSERT(targetCPU != NULL); + + fCore = targetCore; + return rescheduleNeeded; +} + + +bigtime_t +ThreadData::ComputeQuantum() +{ + bigtime_t quantum; + if (fTimeLeft != 0) + quantum = fTimeLeft; + else + quantum = _GetBaseQuantum(); + + if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return quantum; + + quantum += fStolenTime; + fStolenTime = 0; + + int32 threadCount = (fCore->fThreadCount + 1) / fCore->fCPUCount; + threadCount = max_c(threadCount, 1); + + quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum); + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + + fTimeLeft = quantum; + fQuantumStart = system_time(); + + return quantum; +} + + +inline bigtime_t +ThreadData::_GetBaseQuantum() const +{ + int32 priority = GetEffectivePriority(); + + const bigtime_t kQuantum0 = gCurrentMode->base_quantum; + if (priority >= B_URGENT_DISPLAY_PRIORITY) + return kQuantum0; + + const bigtime_t kQuantum1 + = kQuantum0 * gCurrentMode->quantum_multipliers[0]; + if (priority > B_NORMAL_PRIORITY) { + return _ScaleQuantum(kQuantum1, kQuantum0, B_URGENT_DISPLAY_PRIORITY, + B_NORMAL_PRIORITY, priority); + } + + const bigtime_t kQuantum2 + = kQuantum0 * gCurrentMode->quantum_multipliers[1]; + return _ScaleQuantum(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, + B_IDLE_PRIORITY, priority); +} + + +/* static */ bigtime_t +ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, + int32 maxPriority, int32 minPriority, int32 priority) +{ + ASSERT(priority <= maxPriority); + ASSERT(priority >= minPriority); + + bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); + result /= maxPriority - minPriority; + return maxQuantum - result; +} + diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h new file mode 100644 index 0000000000..ffdae08035 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -0,0 +1,360 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_THREAD_H +#define KERNEL_SCHEDULER_THREAD_H + + +#include +#include + +#include "scheduler_common.h" +#include "scheduler_cpu.h" + + +namespace Scheduler { + + +struct ThreadData : public DoublyLinkedListLinkImpl, + RunQueueLinkImpl { +public: + ThreadData(Thread* thread); + + void Init(); + void Init(CoreEntry* core); + + void Dump() const; + + inline bool HasCacheExpired() const; + inline bool ShouldRebalance() const; + + inline int32 GetEffectivePriority() const; + + inline void IncreasePenalty(); + inline void CancelPenalty(); + inline bool ShouldCancelPenalty() const; + + bool ChooseCoreAndCPU(CoreEntry*& targetCore, + CPUEntry*& targetCPU); + + inline void GoesAway(); + + inline void PutBack(); + inline void Enqueue(); + inline bool Dequeue(); + + inline void UpdateActivity(bigtime_t active); + inline void ComputeLoad(); + + inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); + bigtime_t ComputeQuantum(); + + inline Thread* GetThread() const { return fThread; } + inline int32 GetLoad() const { return fLoad; } + + inline CoreEntry* GetCore() const { return fCore; } + inline void UnassignCore() { fCore = NULL; } + + bigtime_t fStolenTime; + bigtime_t fQuantumStart; + bigtime_t fLastInterruptTime; + + bigtime_t fWentSleep; + bigtime_t fWentSleepActive; + int32 fWentSleepCount; + + bool fEnqueued; + +private: + inline int32 _GetPenalty() const; + inline int32 _GetMinimalPriority() const; + + inline CoreEntry* _ChooseCore() const; + inline CPUEntry* _ChooseCPU(CoreEntry* core, + bool& rescheduleNeeded) const; + + inline bigtime_t _GetBaseQuantum() const; + static bigtime_t _ScaleQuantum(bigtime_t maxQuantum, + bigtime_t minQuantum, int32 maxPriority, + int32 minPriority, int32 priority); + + Thread* fThread; + + int32 fPriorityPenalty; + int32 fAdditionalPenalty; + + bigtime_t fTimeLeft; + + bigtime_t fMeasureActiveTime; + bigtime_t fMeasureTime; + int32 fLoad; + + CoreEntry* fCore; +}; + + +inline bool +ThreadData::HasCacheExpired() const +{ + return gCurrentMode->has_cache_expired(this); +} + + +inline bool +ThreadData::ShouldRebalance() const +{ + ASSERT(!gSingleCore); + return gCurrentMode->should_rebalance(this); +} + + +inline int32 +ThreadData::GetEffectivePriority() const +{ + if (thread_is_idle_thread(fThread)) + return B_IDLE_PRIORITY; + if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return fThread->priority; + + int32 effectivePriority = fThread->priority; + effectivePriority -= _GetPenalty(); + + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return effectivePriority; +} + + +inline void +ThreadData::IncreasePenalty() +{ + if (fThread->priority < B_LOWEST_ACTIVE_PRIORITY) + return; + if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return; + + TRACE("increasing thread %ld penalty\n", fThread->id); + + int32 oldPenalty = fPriorityPenalty++; + + ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); + + const int kMinimalPriority = _GetMinimalPriority(); + if (fThread->priority - oldPenalty <= kMinimalPriority) { + fPriorityPenalty = oldPenalty; + fAdditionalPenalty++; + } +} + + +inline void +ThreadData::CancelPenalty() +{ + if (fPriorityPenalty != 0) + TRACE("cancelling thread %ld penalty\n", fThread->id); + + fAdditionalPenalty = 0; + fPriorityPenalty = 0; +} + + +inline bool +ThreadData::ShouldCancelPenalty() const +{ + if (fCore == NULL) + return false; + + return atomic_get(&fCore->fStarvationCounter) != fWentSleepCount + && system_time() - fWentSleep > gCurrentMode->base_quantum; +} + + +inline void +ThreadData::GoesAway() +{ + fLastInterruptTime = 0; + + fWentSleep = system_time(); + fWentSleepActive = atomic_get64(&fCore->fActiveTime); + fWentSleepCount = atomic_get(&fCore->fStarvationCounter); +} + + +inline void +ThreadData::PutBack() +{ + ComputeLoad(); + fWentSleepCount = -1; + + int32 priority = GetEffectivePriority(); + + SpinLocker runQueueLocker(fCore->fQueueLock); + ASSERT(!fEnqueued); + fEnqueued = true; + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->cpu != NULL); + + CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num]; + cpu->fRunQueue.PushFront(this, priority); + } else { + fCore->fRunQueue.PushFront(this, priority); + atomic_add(&fCore->fThreadCount, 1); + } +} + + +inline void +ThreadData::Enqueue() +{ + fThread->state = B_THREAD_READY; + ComputeLoad(); + fWentSleepCount = 0; + + int32 priority = GetEffectivePriority(); + + SpinLocker runQueueLocker(fCore->fQueueLock); + ASSERT(!fEnqueued); + fEnqueued = true; + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->previous_cpu != NULL); + + CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; + cpu->fRunQueue.PushBack(this, priority); + } else { + fCore->fRunQueue.PushBack(this, priority); + fCore->fThreadList.Insert(this); + + atomic_add(&fCore->fThreadCount, 1); + } +} + + +inline bool +ThreadData::Dequeue() +{ + SpinLocker runQueueLocker(fCore->fQueueLock); + if (!fEnqueued) + return false; + + fEnqueued = false; + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->previous_cpu != NULL); + + CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; + cpu->fRunQueue.Remove(this); + } else { + fCore->fRunQueue.Remove(this); + + ASSERT(fWentSleepCount < 1); + if (fWentSleepCount == 0) + fCore->fThreadList.Remove(this); + atomic_add(&fCore->fThreadCount, -1); + } + + return true; +} + + +inline void +ThreadData::UpdateActivity(bigtime_t active) +{ + fMeasureActiveTime += active; + gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; + atomic_add64(&fCore->fActiveTime, active); +} + + +inline void +ThreadData::ComputeLoad() +{ + if (fLastInterruptTime > 0) { + bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; + interruptTime -= fLastInterruptTime; + fMeasureActiveTime -= interruptTime; + } + + compute_load(fMeasureTime, fMeasureActiveTime, fLoad); +} + + +inline bool +ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) +{ + if (hasYielded) { + fTimeLeft = 0; + return true; + } + + bigtime_t timeUsed = system_time() - fQuantumStart; + fTimeLeft -= timeUsed; + fTimeLeft = std::max(fTimeLeft, bigtime_t(0)); + + // too little time left, it's better make the next quantum a bit longer + if (wasPreempted || fTimeLeft <= gCurrentMode->minimal_quantum) { + fStolenTime += fTimeLeft; + fTimeLeft = 0; + } + + return fTimeLeft == 0; +} + + +inline int32 +ThreadData::_GetPenalty() const +{ + int32 penalty = fPriorityPenalty; + + const int kMinimalPriority = _GetMinimalPriority(); + if (kMinimalPriority > 0) + penalty += fAdditionalPenalty % kMinimalPriority; + + return penalty; +} + + +inline int32 +ThreadData::_GetMinimalPriority() const +{ + const int32 kDivisor = 5; + + const int32 kMaximalPriority = 25; + const int32 kMinimalPriority = B_LOWEST_ACTIVE_PRIORITY; + + int32 priority = fThread->priority / kDivisor; + return std::max(std::min(priority, kMaximalPriority), kMinimalPriority); +} + + +inline CoreEntry* +ThreadData::_ChooseCore() const +{ + ASSERT(!gSingleCore); + return gCurrentMode->choose_core(this); +} + + +inline CPUEntry* +ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const +{ + SpinLocker cpuLocker(core->fCPULock); + CPUEntry* cpu = core->fCPUHeap.PeekMinimum(); + ASSERT(cpu != NULL); + + int32 threadPriority = GetEffectivePriority(); + if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { + cpu->UpdatePriority(threadPriority); + rescheduleNeeded = true; + } else + rescheduleNeeded = false; + + return cpu; +} + + +} // namespace Scheduler + + +#endif // KERNEL_SCHEDULER_THREAD_H + From 52b442a687680ddd6a55478baeaa42ec87077f49 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 5 Dec 2013 23:10:04 +0100 Subject: [PATCH 172/273] kernel: smp_cpu_rendezvous(): Use counter instead of bitmap --- headers/private/kernel/smp.h | 2 +- src/system/kernel/arch/x86/arch_cpu.cpp | 18 +++++++++--------- src/system/kernel/main.cpp | 12 ++++++------ src/system/kernel/smp.cpp | 13 ++++++------- 4 files changed, 22 insertions(+), 23 deletions(-) diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 7690a82d64..28a1ccea66 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -47,7 +47,7 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); status_t smp_init_post_generic_syscalls(void); bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); void smp_wake_up_non_boot_cpus(void); -void smp_cpu_rendezvous(uint32 *var, int current_cpu); +void smp_cpu_rendezvous(uint32 *var); void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 16d60bfe96..2046f2f176 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -189,7 +189,7 @@ set_mtrr(void* _parameter, int cpu) = (struct set_mtrr_parameter*)_parameter; // wait until all CPUs have arrived here - smp_cpu_rendezvous(&sCpuRendezvous, cpu); + smp_cpu_rendezvous(&sCpuRendezvous); // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU // that initiated the call_all_cpus() from doing that again and clearing @@ -206,8 +206,8 @@ set_mtrr(void* _parameter, int cpu) enable_caches(); // wait until all CPUs have arrived here - smp_cpu_rendezvous(&sCpuRendezvous2, cpu); - smp_cpu_rendezvous(&sCpuRendezvous3, cpu); + smp_cpu_rendezvous(&sCpuRendezvous2); + smp_cpu_rendezvous(&sCpuRendezvous3); } @@ -217,7 +217,7 @@ set_mtrrs(void* _parameter, int cpu) set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter; // wait until all CPUs have arrived here - smp_cpu_rendezvous(&sCpuRendezvous, cpu); + smp_cpu_rendezvous(&sCpuRendezvous); // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU // that initiated the call_all_cpus() from doing that again and clearing @@ -234,8 +234,8 @@ set_mtrrs(void* _parameter, int cpu) enable_caches(); // wait until all CPUs have arrived here - smp_cpu_rendezvous(&sCpuRendezvous2, cpu); - smp_cpu_rendezvous(&sCpuRendezvous3, cpu); + smp_cpu_rendezvous(&sCpuRendezvous2); + smp_cpu_rendezvous(&sCpuRendezvous3); } @@ -243,7 +243,7 @@ static void init_mtrrs(void* _unused, int cpu) { // wait until all CPUs have arrived here - smp_cpu_rendezvous(&sCpuRendezvous, cpu); + smp_cpu_rendezvous(&sCpuRendezvous); // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU // that initiated the call_all_cpus() from doing that again and clearing @@ -259,8 +259,8 @@ init_mtrrs(void* _unused, int cpu) enable_caches(); // wait until all CPUs have arrived here - smp_cpu_rendezvous(&sCpuRendezvous2, cpu); - smp_cpu_rendezvous(&sCpuRendezvous3, cpu); + smp_cpu_rendezvous(&sCpuRendezvous2); + smp_cpu_rendezvous(&sCpuRendezvous3); } diff --git a/src/system/kernel/main.cpp b/src/system/kernel/main.cpp index f13526542b..4350841f1c 100644 --- a/src/system/kernel/main.cpp +++ b/src/system/kernel/main.cpp @@ -100,13 +100,13 @@ _start(kernel_args *bootKernelArgs, int currentCPU) smp_set_num_cpus(bootKernelArgs->num_cpus); // wait for all the cpus to get here - smp_cpu_rendezvous(&sCpuRendezvous, currentCPU); + smp_cpu_rendezvous(&sCpuRendezvous); // the passed in kernel args are in a non-allocated range of memory if (currentCPU == 0) memcpy(&sKernelArgs, bootKernelArgs, sizeof(kernel_args)); - smp_cpu_rendezvous(&sCpuRendezvous2, currentCPU); + smp_cpu_rendezvous(&sCpuRendezvous2); // do any pre-booting cpu config cpu_preboot_init_percpu(&sKernelArgs, currentCPU); @@ -221,13 +221,13 @@ _start(kernel_args *bootKernelArgs, int currentCPU) TRACE("waking up AP cpus\n"); sCpuRendezvous = sCpuRendezvous2 = 0; smp_wake_up_non_boot_cpus(); - smp_cpu_rendezvous(&sCpuRendezvous, 0); // wait until they're booted + smp_cpu_rendezvous(&sCpuRendezvous); // wait until they're booted // exit the kernel startup phase (mutexes, etc work from now on out) TRACE("exiting kernel startup\n"); gKernelStartup = false; - smp_cpu_rendezvous(&sCpuRendezvous2, 0); + smp_cpu_rendezvous(&sCpuRendezvous2); // release the AP cpus to go enter the scheduler TRACE("starting scheduler on cpu 0 and enabling interrupts\n"); @@ -244,8 +244,8 @@ _start(kernel_args *bootKernelArgs, int currentCPU) smp_per_cpu_init(&sKernelArgs, currentCPU); // wait for all other AP cpus to get to this point - smp_cpu_rendezvous(&sCpuRendezvous, currentCPU); - smp_cpu_rendezvous(&sCpuRendezvous2, currentCPU); + smp_cpu_rendezvous(&sCpuRendezvous); + smp_cpu_rendezvous(&sCpuRendezvous2); // welcome to the machine scheduler_start(); diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 9788b2b343..e54bc38170 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -1313,11 +1313,11 @@ bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous) { if (cpu == 0) { - smp_cpu_rendezvous(rendezVous, cpu); + smp_cpu_rendezvous(rendezVous); return true; } - smp_cpu_rendezvous(rendezVous, cpu); + smp_cpu_rendezvous(rendezVous); while (atomic_get(&sBootCPUSpin) == 0) { if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0) @@ -1353,13 +1353,12 @@ smp_wake_up_non_boot_cpus() ensured via another rendez-vous) the variable can be reset. */ void -smp_cpu_rendezvous(uint32* var, int current_cpu) +smp_cpu_rendezvous(uint32* var) { - atomic_or((int32*)var, 1 << current_cpu); + atomic_add((int32*)var, 1); - uint32 allReady = ((uint32)1 << sNumCPUs) - 1; - while ((uint32)atomic_get((int32*)var) != allReady) - cpu_wait((int32*)var, allReady); + while ((uint32)atomic_get((int32*)var) < sNumCPUs) + cpu_wait((int32*)var, sNumCPUs); } From 730882d6153a57265ebb03c0304e54b94658cf1f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 5 Dec 2013 23:13:30 +0100 Subject: [PATCH 173/273] kernel: Remove RunQueueLink.h --- headers/private/kernel/RunQueueLink.h | 56 -------------------------- src/system/kernel/scheduler/RunQueue.h | 43 +++++++++++++++++++- 2 files changed, 42 insertions(+), 57 deletions(-) delete mode 100644 headers/private/kernel/RunQueueLink.h diff --git a/headers/private/kernel/RunQueueLink.h b/headers/private/kernel/RunQueueLink.h deleted file mode 100644 index 79fdc60a1f..0000000000 --- a/headers/private/kernel/RunQueueLink.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2013 Haiku, Inc. All rights reserved. - * Distributed under the terms of the MIT License. - * - * Authors: - * Paweł Dziepak, pdziepak@quarnos.org - */ -#ifndef RUN_QUEUE_LINK_H -#define RUN_QUEUE_LINK_H - - -template -struct RunQueueLink { - RunQueueLink(); - - unsigned int fPriority; - Element* fPrevious; - Element* fNext; -}; - -template -class RunQueueLinkImpl { -public: - inline RunQueueLink* GetRunQueueLink(); - -private: - RunQueueLink fRunQueueLink; -}; - - -#if KDEBUG -template -RunQueueLink::RunQueueLink() - : - fPrevious(NULL), - fNext(NULL) -{ -} -#else -template -RunQueueLink::RunQueueLink() -{ -} -#endif - - -template -RunQueueLink* -RunQueueLinkImpl::GetRunQueueLink() -{ - return &fRunQueueLink; -} - - -#endif // RUN_QUEUE_LINK_H - diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index a38fb32f5d..4ecda7227a 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -9,10 +9,27 @@ #define RUN_QUEUE_H -#include #include +template +struct RunQueueLink { + RunQueueLink(); + + unsigned int fPriority; + Element* fPrevious; + Element* fNext; +}; + +template +class RunQueueLinkImpl { +public: + inline RunQueueLink* GetRunQueueLink(); + +private: + RunQueueLink fRunQueueLink; +}; + template class RunQueueStandardGetLink { private: @@ -90,6 +107,30 @@ private: }; +#if KDEBUG +template +RunQueueLink::RunQueueLink() + : + fPrevious(NULL), + fNext(NULL) +{ +} +#else +template +RunQueueLink::RunQueueLink() +{ +} +#endif + + +template +RunQueueLink* +RunQueueLinkImpl::GetRunQueueLink() +{ + return &fRunQueueLink; +} + + template RunQueueLink* RunQueueStandardGetLink::operator()(Element* element) const From 5f3798921f1bde57a28d7a1704dd22314bbf9c24 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 5 Dec 2013 23:33:03 +0100 Subject: [PATCH 174/273] x86/paging: Use ROUNUP() to compute the number of initial slots --- .../kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp | 2 +- src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp index 0f0272f6ba..37f5149d7f 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp @@ -38,7 +38,7 @@ #define MAX_INITIAL_POOLS \ - ((SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS + 1023) / 1024) + (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, 1024) / 1024) using X86LargePhysicalPageMapper::PhysicalPageSlot; diff --git a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp index e96b0be792..38d4e0e676 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp +++ b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp @@ -42,8 +42,9 @@ #define MAX_INITIAL_POOLS \ - ((SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS \ - + kPAEPageTableEntryCount - 1) / kPAEPageTableEntryCount) + (ROUNDUP(SMP_MAX_CPUS * TOTAL_SLOTS_PER_CPU + EXTRA_SLOTS, \ + kPAEPageTableEntryCount) \ + / kPAEPageTableEntryCount) using X86LargePhysicalPageMapper::PhysicalPageSlot; From 7629d527c5ee0f402c5a16d0f42c2b79a5571b07 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 02:54:34 +0100 Subject: [PATCH 175/273] kernel: Use CPUSet in ICI code instead of cpu_mask_t --- headers/private/kernel/arch/smp.h | 3 - headers/private/kernel/smp.h | 83 ++++++++++++++++++- src/system/kernel/arch/x86/arch_thread.cpp | 5 +- .../arch/x86/paging/X86PagingStructures.cpp | 3 +- .../arch/x86/paging/X86PagingStructures.h | 4 +- .../arch/x86/paging/X86VMTranslationMap.cpp | 14 ++-- src/system/kernel/smp.cpp | 63 ++++++++------ 7 files changed, 131 insertions(+), 44 deletions(-) diff --git a/headers/private/kernel/arch/smp.h b/headers/private/kernel/arch/smp.h index e8a66b97da..f85fcf136f 100644 --- a/headers/private/kernel/arch/smp.h +++ b/headers/private/kernel/arch/smp.h @@ -11,9 +11,6 @@ struct kernel_args; -// must match MAX_BOOT_CPUS in platform_kernel_args.h -#define SMP_MAX_CPUS MAX_BOOT_CPUS - #ifdef __cplusplus extern "C" { #endif diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 28a1ccea66..0452592abb 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -9,8 +9,17 @@ #define KERNEL_SMP_H +#include +#include + #include +#include + + +#define SMP_MAX_CPUS MAX_BOOT_CPUS + + struct kernel_args; @@ -35,6 +44,26 @@ typedef uint32 cpu_mask_t; typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3); +class CPUSet { +public: + CPUSet() { ClearAll(); } + + inline void ClearAll(); + inline void SetAll(); + + inline void SetBit(int32 cpu); + inline void ClearBit(int32 cpu); + + inline bool GetBit(int32 cpu) const; + + inline bool IsEmpty() const; + +private: + static const int kArraySize = ROUNDUP(SMP_MAX_CPUS, 32) / 32; + + uint32 fBitmap[kArraySize]; +}; + #ifdef __cplusplus extern "C" { @@ -47,10 +76,10 @@ status_t smp_per_cpu_init(struct kernel_args *args, int32 cpu); status_t smp_init_post_generic_syscalls(void); bool smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous); void smp_wake_up_non_boot_cpus(void); -void smp_cpu_rendezvous(uint32 *var); +void smp_cpu_rendezvous(uint32* var); void smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); -void smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, +void smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); void smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, void *data_ptr, uint32 flags); @@ -68,6 +97,56 @@ int smp_intercpu_int_handler(int32 cpu); #endif +inline void +CPUSet::ClearAll() +{ + memset(fBitmap, 0, sizeof(fBitmap)); +} + + +inline void +CPUSet::SetAll() +{ + memset(fBitmap, ~uint8(0), sizeof(fBitmap)); +} + + +inline void +CPUSet::SetBit(int32 cpu) +{ + uint32* element = &fBitmap[cpu % kArraySize]; + atomic_or(element, 1u << (cpu / kArraySize)); +} + + +inline void +CPUSet::ClearBit(int32 cpu) +{ + uint32* element = &fBitmap[cpu % kArraySize]; + atomic_and(element, ~uint32(1u << (cpu / kArraySize))); +} + + +inline bool +CPUSet::GetBit(int32 cpu) const +{ + int32* element = (int32*)&fBitmap[cpu % kArraySize]; + return ((uint32)atomic_get(element) & (1u << (cpu / kArraySize))) != 0; +} + + +inline bool +CPUSet::IsEmpty() const +{ + for (int i = 0; i < kArraySize; i++) { + if (fBitmap[i] != 0) + return false; + } + + return true; +} + + // Unless spinlock debug features are enabled, try to inline // {acquire,release}_spinlock(). #if !DEBUG_SPINLOCKS && !B_DEBUG_SPINLOCK_CONTENTION diff --git a/src/system/kernel/arch/x86/arch_thread.cpp b/src/system/kernel/arch/x86/arch_thread.cpp index e7b1298a59..9fbbaa3ad3 100644 --- a/src/system/kernel/arch/x86/arch_thread.cpp +++ b/src/system/kernel/arch/x86/arch_thread.cpp @@ -229,9 +229,8 @@ arch_thread_context_switch(Thread* from, Thread* to) != activePagingStructures) { // update on which CPUs the address space is used int cpu = cpuData->cpu_num; - atomic_and(&activePagingStructures->active_on_cpus, - ~((uint32)1 << cpu)); - atomic_or(&toPagingStructures->active_on_cpus, (uint32)1 << cpu); + activePagingStructures->active_on_cpus.ClearBit(cpu); + toPagingStructures->active_on_cpus.SetBit(cpu); // assign the new paging structures to the CPU toPagingStructures->AddReference(); diff --git a/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp b/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp index 4a2d9ee9ad..2844391341 100644 --- a/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp +++ b/src/system/kernel/arch/x86/paging/X86PagingStructures.cpp @@ -9,8 +9,7 @@ X86PagingStructures::X86PagingStructures() : - ref_count(1), - active_on_cpus(0) + ref_count(1) { } diff --git a/src/system/kernel/arch/x86/paging/X86PagingStructures.h b/src/system/kernel/arch/x86/paging/X86PagingStructures.h index 6fd882b39f..2744b0cef0 100644 --- a/src/system/kernel/arch/x86/paging/X86PagingStructures.h +++ b/src/system/kernel/arch/x86/paging/X86PagingStructures.h @@ -14,11 +14,13 @@ #include +#include + struct X86PagingStructures : DeferredDeletable { phys_addr_t pgdir_phys; int32 ref_count; - int32 active_on_cpus; + CPUSet active_on_cpus; // mask indicating on which CPUs the map is currently used X86PagingStructures(); diff --git a/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp b/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp index a9768653ef..5ed05001ea 100644 --- a/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp +++ b/src/system/kernel/arch/x86/paging/X86VMTranslationMap.cpp @@ -113,9 +113,10 @@ X86VMTranslationMap::Flush() restore_interrupts(state); int cpu = smp_get_current_cpu(); - uint32 cpuMask = PagingStructures()->active_on_cpus - & ~((uint32)1 << cpu); - if (cpuMask != 0) { + CPUSet cpuMask = PagingStructures()->active_on_cpus; + cpuMask.ClearBit(cpu); + + if (!cpuMask.IsEmpty()) { smp_send_multicast_ici(cpuMask, SMP_MSG_USER_INVALIDATE_PAGES, 0, 0, 0, NULL, SMP_MSG_FLAG_SYNC); } @@ -132,9 +133,10 @@ X86VMTranslationMap::Flush() SMP_MSG_FLAG_SYNC); } else { int cpu = smp_get_current_cpu(); - uint32 cpuMask = PagingStructures()->active_on_cpus - & ~((uint32)1 << cpu); - if (cpuMask != 0) { + CPUSet cpuMask = PagingStructures()->active_on_cpus; + cpuMask.ClearBit(cpu); + + if (!cpuMask.IsEmpty()) { smp_send_multicast_ici(cpuMask, SMP_MSG_INVALIDATE_PAGE_LIST, (addr_t)fInvalidPages, fInvalidPagesCount, 0, NULL, SMP_MSG_FLAG_SYNC); diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index e54bc38170..63d280569d 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -80,7 +80,7 @@ struct smp_msg { uint32 flags; int32 ref_count; int32 done; - uint32 proc_bitmap; + CPUSet proc_bitmap; }; enum mailbox_source { @@ -90,7 +90,8 @@ enum mailbox_source { static int32 sBootCPUSpin = 0; -static int32 sEarlyCPUCall = 0; +static int32 sEarlyCPUCallCount; +static CPUSet sEarlyCPUCallSet; static void (*sEarlyCPUCallFunction)(void*, int); void* sEarlyCPUCallCookie; @@ -294,7 +295,13 @@ dump_ici_message(int argc, char** argv) kprintf(" flags: %" B_PRIx32 "\n", message->flags); kprintf(" ref_count: %" B_PRIx32 "\n", message->ref_count); kprintf(" done: %s\n", message->done == 1 ? "true" : "false"); - kprintf(" proc_bitmap: %" B_PRIx32 "\n", message->proc_bitmap); + + kprintf(" proc_bitmap: "); + for (int32 i = 0; i < sNumCPUs; i++) { + if (message->proc_bitmap.GetBit(i)) + kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i); + } + kprintf("\n"); return 0; } @@ -809,14 +816,14 @@ check_for_message(int currentCPU, mailbox_source& sourceMailbox) msg = sBroadcastMessages; while (msg != NULL) { - if (CHECK_BIT(msg->proc_bitmap, currentCPU) != 0) { + if (!msg->proc_bitmap.GetBit(currentCPU)) { // we have handled this one already msg = msg->next; continue; } // mark it so we wont try to process this one again - msg->proc_bitmap = SET_BIT(msg->proc_bitmap, currentCPU); + msg->proc_bitmap.ClearBit(currentCPU); atomic_add(&gCPU[currentCPU].ici_counter, 1); sourceMailbox = MAILBOX_BCAST; @@ -994,7 +1001,8 @@ static void process_early_cpu_call(int32 cpu) { sEarlyCPUCallFunction(sEarlyCPUCallCookie, cpu); - atomic_and(&sEarlyCPUCall, ~(uint32)(1 << cpu)); + sEarlyCPUCallSet.ClearBit(cpu); + atomic_add(&sEarlyCPUCallCount, 1); } @@ -1005,14 +1013,13 @@ call_all_cpus_early(void (*function)(void*, int), void* cookie) sEarlyCPUCallFunction = function; sEarlyCPUCallCookie = cookie; - uint32 cpuMask = (1 << sNumCPUs) - 2; - // all CPUs but the boot cpu - - atomic_set(&sEarlyCPUCall, cpuMask); + atomic_set(&sEarlyCPUCallCount, 1); + sEarlyCPUCallSet.SetAll(); + sEarlyCPUCallSet.ClearBit(0); // wait for all CPUs to finish - while ((atomic_get(&sEarlyCPUCall) & cpuMask) != 0) - cpu_pause(); + while (atomic_get(&sEarlyCPUCallCount) < sNumCPUs) + cpu_wait(&sEarlyCPUCallCount, sNumCPUs); } function(cookie, 0); @@ -1099,24 +1106,16 @@ smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, void -smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, +smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, addr_t data2, addr_t data3, void *dataPointer, uint32 flags) { if (!sICIEnabled) return; - int currentCPU = smp_get_current_cpu(); - cpuMask &= ~((cpu_mask_t)1 << currentCPU) - & (((cpu_mask_t)1 << sNumCPUs) - 1); - if (cpuMask == 0) { - panic("smp_send_multicast_ici(): 0 CPU mask"); - return; - } - // count target CPUs int32 targetCPUs = 0; for (int32 i = 0; i < sNumCPUs; i++) { - if ((cpuMask & (cpu_mask_t)1 << i) != 0) + if (cpuMask.GetBit(i)) targetCPUs++; } @@ -1131,9 +1130,17 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, msg->data_ptr = dataPointer; msg->ref_count = targetCPUs; msg->flags = flags; - msg->proc_bitmap = ~cpuMask; msg->done = 0; + int currentCPU = smp_get_current_cpu(); + msg->proc_bitmap = cpuMask; + msg->proc_bitmap.ClearBit(currentCPU); + if (msg->proc_bitmap.IsEmpty()) { + panic("smp_send_multicast_ici(): 0 CPU mask"); + return; + } + + // stick it in the broadcast mailbox acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); msg->next = sBroadcastMessages; @@ -1142,7 +1149,7 @@ smp_send_multicast_ici(cpu_mask_t cpuMask, int32 message, addr_t data, atomic_add(&sBroadcastMessageCounter, 1); for (int32 i = 0; i < sNumCPUs; i++) { - if ((cpuMask & (cpu_mask_t)1 << i) == 0) + if (!cpuMask.GetBit(i)) atomic_add(&gCPU[i].ici_counter, 1); } @@ -1193,7 +1200,8 @@ smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, msg->data_ptr = dataPointer; msg->ref_count = sNumCPUs - 1; msg->flags = flags; - msg->proc_bitmap = SET_BIT(0, currentCPU); + msg->proc_bitmap.SetAll(); + msg->proc_bitmap.ClearBit(currentCPU); msg->done = 0; TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast " @@ -1258,7 +1266,8 @@ smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, msg->data_ptr = dataPointer; msg->ref_count = sNumCPUs - 1; msg->flags = flags; - msg->proc_bitmap = SET_BIT(0, currentCPU); + msg->proc_bitmap.SetAll(); + msg->proc_bitmap.ClearBit(currentCPU); msg->done = 0; TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " @@ -1320,7 +1329,7 @@ smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous) smp_cpu_rendezvous(rendezVous); while (atomic_get(&sBootCPUSpin) == 0) { - if ((atomic_get(&sEarlyCPUCall) & (1 << cpu)) != 0) + if (sEarlyCPUCallSet.GetBit(cpu)) process_early_cpu_call(cpu); cpu_pause(); From 5c7f09c4d77dd52f0432c084fa3ce13001f8f1cd Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 03:10:50 +0100 Subject: [PATCH 176/273] kernel/util: Bitmap: Silence warning --- src/system/kernel/util/Bitmap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/util/Bitmap.cpp b/src/system/kernel/util/Bitmap.cpp index 588e996006..8f47c95e19 100644 --- a/src/system/kernel/util/Bitmap.cpp +++ b/src/system/kernel/util/Bitmap.cpp @@ -60,7 +60,7 @@ Bitmap::GetHighestSet() const if (sizeof(addr_t) == sizeof(uint32)) return log2(fBits[i]) + i * kBitsPerElement; - uint32 v = fBits[i] >> 32; + uint32 v = (uint64)fBits[i] >> 32; if (v != 0) return log2(v) + sizeof(uint32) * 8 + i * kBitsPerElement; return log2(fBits[i]) + i * kBitsPerElement; From e052b3e630aa8ef3520cfc5462f06fd584a0be65 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 03:11:42 +0100 Subject: [PATCH 177/273] x86/paging: Remove unusued local variable --- src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp index 37f5149d7f..e8e0419b07 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp @@ -332,7 +332,6 @@ X86PagingMethod32Bit::InitPostArea(kernel_args* args) // now that the vm is initialized, create an area that represents // the page hole void *temp; - status_t error; area_id area; // unmap the page hole hack we were using before From 2b04d8ab318733b3eb6d66dc5bb25e441fbca3c1 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 03:25:19 +0100 Subject: [PATCH 178/273] x86/paging: Use CPUSet instead of cpu_mask_t --- headers/private/kernel/smp.h | 2 -- .../x86/paging/x86_physical_page_mapper_large_memory.cpp | 9 +++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 0452592abb..0085767ee5 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -40,8 +40,6 @@ enum { SMP_MSG_FLAG_FREE_ARG = 0x2, }; -typedef uint32 cpu_mask_t; - typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr_t data3); class CPUSet { diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp index 0b75b14dc4..78301a7d4a 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp @@ -105,7 +105,7 @@ private: struct page_slot { PhysicalPageSlot* slot; phys_addr_t physicalAddress; - cpu_mask_t valid; + CPUSet valid; }; page_slot fSlots[SLOTS_PER_TRANSLATION_MAP]; @@ -389,10 +389,10 @@ LargeMemoryTranslationMapPhysicalPageMapper::GetPageTableAt( page_slot& slot = fSlots[i]; if (slot.physicalAddress == physicalAddress) { fNextSlot = (i + 1) & (fSlotCount - 1); - if ((slot.valid & (1 << currentCPU)) == 0) { + if (!slot.valid.GetBit(currentCPU)) { // not valid on this CPU -- invalidate the TLB entry invalidate_TLB(slot.slot->address); - slot.valid |= 1 << currentCPU; + slot.valid.SetBit(currentCPU); } return (void*)slot.slot->address; } @@ -404,7 +404,8 @@ LargeMemoryTranslationMapPhysicalPageMapper::GetPageTableAt( slot.physicalAddress = physicalAddress; slot.slot->Map(physicalAddress); - slot.valid = 1 << currentCPU; + slot.valid.ClearAll(); + slot.valid.SetBit(currentCPU); return (void*)slot.slot->address; } From f0fe98176905609bd112ff077dfa3fe8545b459f Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 03:31:20 +0100 Subject: [PATCH 179/273] kernel: Fix gcc2 build --- headers/private/kernel/smp.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 0085767ee5..1f4b0f4298 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -112,7 +112,7 @@ CPUSet::SetAll() inline void CPUSet::SetBit(int32 cpu) { - uint32* element = &fBitmap[cpu % kArraySize]; + int32* element = (int32*)&fBitmap[cpu % kArraySize]; atomic_or(element, 1u << (cpu / kArraySize)); } @@ -120,7 +120,7 @@ CPUSet::SetBit(int32 cpu) inline void CPUSet::ClearBit(int32 cpu) { - uint32* element = &fBitmap[cpu % kArraySize]; + int32* element = (int32*)&fBitmap[cpu % kArraySize]; atomic_and(element, ~uint32(1u << (cpu / kArraySize))); } From 3e0e3be7604ed12ab61b58789c44bc6d7333f48b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 19:43:08 +0100 Subject: [PATCH 180/273] boot, kernel: Replace MAX_BOOT_CPUS with SMP_MAX_CPUS --- headers/private/kernel/arch/x86/arch_kernel_args.h | 4 ++-- headers/private/kernel/boot/kernel_args.h | 2 +- .../boot/platform/bios_ia32/platform_kernel_args.h | 4 ++-- headers/private/kernel/smp.h | 3 --- src/system/boot/platform/bios_ia32/smp.cpp | 12 ++++++------ src/system/kernel/cpu.cpp | 2 +- src/system/kernel/smp.cpp | 4 ++-- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_kernel_args.h b/headers/private/kernel/arch/x86/arch_kernel_args.h index 7e6c967582..a0dbae7bdb 100644 --- a/headers/private/kernel/arch/x86/arch_kernel_args.h +++ b/headers/private/kernel/arch/x86/arch_kernel_args.h @@ -40,8 +40,8 @@ typedef struct { uint32 apic_phys; FixedWidthPointer apic; uint32 ioapic_phys; - uint32 cpu_apic_id[MAX_BOOT_CPUS]; - uint32 cpu_apic_version[MAX_BOOT_CPUS]; + uint32 cpu_apic_id[SMP_MAX_CPUS]; + uint32 cpu_apic_version[SMP_MAX_CPUS]; // hpet stuff uint32 hpet_phys; FixedWidthPointer hpet; diff --git a/headers/private/kernel/boot/kernel_args.h b/headers/private/kernel/boot/kernel_args.h index 95542b1cd3..1ebd5c1ef4 100644 --- a/headers/private/kernel/boot/kernel_args.h +++ b/headers/private/kernel/boot/kernel_args.h @@ -59,7 +59,7 @@ typedef struct kernel_args { uint64 ignored_physical_memory; uint32 num_cpus; - addr_range cpu_kstack[MAX_BOOT_CPUS]; + addr_range cpu_kstack[SMP_MAX_CPUS]; // boot volume KMessage data FixedWidthPointer boot_volume; diff --git a/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h b/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h index a191a0db14..6772222f01 100644 --- a/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h +++ b/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h @@ -15,8 +15,8 @@ #include -// must match SMP_MAX_CPUS in arch_smp.h -#define MAX_BOOT_CPUS 8 +#define SMP_MAX_CPUS 8 + #define MAX_PHYSICAL_MEMORY_RANGE 32 #define MAX_PHYSICAL_ALLOCATED_RANGE 32 #define MAX_VIRTUAL_ALLOCATED_RANGE 32 diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 1f4b0f4298..88a89e4064 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -17,9 +17,6 @@ #include -#define SMP_MAX_CPUS MAX_BOOT_CPUS - - struct kernel_args; diff --git a/src/system/boot/platform/bios_ia32/smp.cpp b/src/system/boot/platform/bios_ia32/smp.cpp index a0ed686415..9a2e77ea01 100644 --- a/src/system/boot/platform/bios_ia32/smp.cpp +++ b/src/system/boot/platform/bios_ia32/smp.cpp @@ -158,9 +158,9 @@ smp_do_mp_config(mp_floating_struct *floatingStruct) = (struct mp_base_processor *)pointer; pointer += sizeof(struct mp_base_processor); - if (gKernelArgs.num_cpus == MAX_BOOT_CPUS) { - TRACE(("smp: already reached maximum boot CPUs (%d)\n", - MAX_BOOT_CPUS)); + if (gKernelArgs.num_cpus == SMP_MAX_CPUS) { + TRACE(("smp: already reached maximum CPUs (%d)\n", + SMP_MAX_CPUS)); continue; } @@ -277,9 +277,9 @@ smp_do_acpi_config(void) switch (apic->type) { case ACPI_MADT_LOCAL_APIC: { - if (gKernelArgs.num_cpus == MAX_BOOT_CPUS) { - TRACE(("smp: already reached maximum boot CPUs (%d)\n", - MAX_BOOT_CPUS)); + if (gKernelArgs.num_cpus == SMP_MAX_CPUS) { + TRACE(("smp: already reached maximum CPUs (%d)\n", + SMP_MAX_CPUS)); break; } diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 2fdf6fd1da..3a3df3a043 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -25,7 +25,7 @@ /* global per-cpu structure */ -cpu_ent gCPU[MAX_BOOT_CPUS]; +cpu_ent gCPU[SMP_MAX_CPUS]; uint32 gCPUCacheLevelCount; static cpu_topology_node sCPUTopology; diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 63d280569d..9e987a5212 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -188,9 +188,9 @@ dump_spinlock(int argc, char** argv) static struct { spinlock *lock; bigtime_t timestamp; -} sLatency[B_MAX_CPU_COUNT][NUM_LATENCY_LOCKS]; +} sLatency[SMP_MAX_CPUS][NUM_LATENCY_LOCKS]; -static int32 sLatencyIndex[B_MAX_CPU_COUNT]; +static int32 sLatencyIndex[SMP_MAX_CPUS]; static bool sEnableLatencyCheck; From e6ea745e8114d5fa05ab0ccab92c070e2dcdcfb7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 19:53:56 +0100 Subject: [PATCH 181/273] kernel: Use SMP_MAX_CPUS instead of B_MAX_CPU_COUNT --- src/system/kernel/arch/x86/32/descriptors.cpp | 2 +- src/system/kernel/arch/x86/arch_smp.cpp | 4 ++-- .../x86/paging/x86_physical_page_mapper_large_memory.cpp | 2 +- src/system/kernel/debug/debug.cpp | 4 ++-- src/system/kernel/debug/system_profiler.cpp | 4 ++-- src/system/kernel/debug/user_debugger.cpp | 2 +- src/system/kernel/heap.cpp | 6 +++--- src/system/kernel/thread.cpp | 2 +- src/system/kernel/timer.cpp | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/system/kernel/arch/x86/32/descriptors.cpp b/src/system/kernel/arch/x86/32/descriptors.cpp index d215dc0ffb..53b60e4df8 100644 --- a/src/system/kernel/arch/x86/32/descriptors.cpp +++ b/src/system/kernel/arch/x86/32/descriptors.cpp @@ -28,7 +28,7 @@ #include "interrupts.h" -static interrupt_descriptor* sIDTs[B_MAX_CPU_COUNT]; +static interrupt_descriptor* sIDTs[SMP_MAX_CPUS]; // table with functions handling respective interrupts typedef void interrupt_handler_function(struct iframe* frame); diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index 340854bb79..0a97a97e51 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -34,8 +34,8 @@ #endif -static uint32 sCPUAPICIds[B_MAX_CPU_COUNT]; -static uint32 sAPICVersions[B_MAX_CPU_COUNT]; +static uint32 sCPUAPICIds[SMP_MAX_CPUS]; +static uint32 sAPICVersions[SMP_MAX_CPUS]; static int32 diff --git a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp index 78301a7d4a..07c9446fdc 100644 --- a/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp +++ b/src/system/kernel/arch/x86/paging/x86_physical_page_mapper_large_memory.cpp @@ -168,7 +168,7 @@ private: PhysicalPageSlot* fDebugSlot; PhysicalPageSlotPool* fInitialPool; LargeMemoryTranslationMapPhysicalPageMapper fKernelMapper; - PhysicalPageOpsCPUData fPerCPUData[B_MAX_CPU_COUNT]; + PhysicalPageOpsCPUData fPerCPUData[SMP_MAX_CPUS]; }; static LargeMemoryPhysicalPageMapper sPhysicalPageMapper; diff --git a/src/system/kernel/debug/debug.cpp b/src/system/kernel/debug/debug.cpp index b56145b893..da74437805 100644 --- a/src/system/kernel/debug/debug.cpp +++ b/src/system/kernel/debug/debug.cpp @@ -81,7 +81,7 @@ void call_modules_hook(bool enter); static void syslog_write(const char* text, int32 length, bool notify); -static arch_debug_registers sDebugRegisters[B_MAX_CPU_COUNT]; +static arch_debug_registers sDebugRegisters[SMP_MAX_CPUS]; static debug_page_fault_info sPageFaultInfo; @@ -143,7 +143,7 @@ static int32 sInDebugger = 0; static bool sPreviousDprintfState; static volatile bool sHandOverKDL = false; static int32 sHandOverKDLToCPU = -1; -static bool sCPUTrapped[B_MAX_CPU_COUNT]; +static bool sCPUTrapped[SMP_MAX_CPUS]; #define distance(a, b) ((a) < (b) ? (b) - (a) : (a) - (b)) diff --git a/src/system/kernel/debug/system_profiler.cpp b/src/system/kernel/debug/system_profiler.cpp index 5b341b2254..af3570737c 100644 --- a/src/system/kernel/debug/system_profiler.cpp +++ b/src/system/kernel/debug/system_profiler.cpp @@ -199,8 +199,8 @@ private: bool fWaitObjectNotificationsRequested; Thread* volatile fWaitingProfilerThread; bool fProfilingActive; - bool fReentered[B_MAX_CPU_COUNT]; - CPUProfileData fCPUData[B_MAX_CPU_COUNT]; + bool fReentered[SMP_MAX_CPUS]; + CPUProfileData fCPUData[SMP_MAX_CPUS]; WaitObject* fWaitObjectBuffer; int32 fWaitObjectCount; WaitObjectList fUsedWaitObjects; diff --git a/src/system/kernel/debug/user_debugger.cpp b/src/system/kernel/debug/user_debugger.cpp index 6999556c04..a4eb75ebc3 100644 --- a/src/system/kernel/debug/user_debugger.cpp +++ b/src/system/kernel/debug/user_debugger.cpp @@ -53,7 +53,7 @@ static port_id sDefaultDebuggerPort = -1; // accessed atomically -static timer sProfilingTimers[B_MAX_CPU_COUNT]; +static timer sProfilingTimers[SMP_MAX_CPUS]; // a profiling timer for each CPU -- used when a profiled thread is running // on that CPU diff --git a/src/system/kernel/heap.cpp b/src/system/kernel/heap.cpp index a2815bff6a..9a6ab41571 100644 --- a/src/system/kernel/heap.cpp +++ b/src/system/kernel/heap.cpp @@ -187,9 +187,9 @@ static const heap_class sHeapClasses[HEAP_CLASS_COUNT] = { static uint32 sHeapCount; -static heap_allocator *sHeaps[HEAP_CLASS_COUNT * B_MAX_CPU_COUNT]; -static uint32 *sLastGrowRequest[HEAP_CLASS_COUNT * B_MAX_CPU_COUNT]; -static uint32 *sLastHandledGrowRequest[HEAP_CLASS_COUNT * B_MAX_CPU_COUNT]; +static heap_allocator *sHeaps[HEAP_CLASS_COUNT * SMP_MAX_CPUS]; +static uint32 *sLastGrowRequest[HEAP_CLASS_COUNT * SMP_MAX_CPUS]; +static uint32 *sLastHandledGrowRequest[HEAP_CLASS_COUNT * SMP_MAX_CPUS]; static heap_allocator *sVIPHeap; static heap_allocator *sGrowHeap = NULL; diff --git a/src/system/kernel/thread.cpp b/src/system/kernel/thread.cpp index fc6cbe3842..6f9d4da1c9 100644 --- a/src/system/kernel/thread.cpp +++ b/src/system/kernel/thread.cpp @@ -70,7 +70,7 @@ typedef BKernel::TeamThreadTable ThreadHashTable; // thread list -static Thread sIdleThreads[B_MAX_CPU_COUNT]; +static Thread sIdleThreads[SMP_MAX_CPUS]; static ThreadHashTable sThreadHash; static spinlock sThreadHashLock = B_SPINLOCK_INITIALIZER; static thread_id sNextThreadID = 2; diff --git a/src/system/kernel/timer.cpp b/src/system/kernel/timer.cpp index 5a3ea20969..28284f4ae1 100644 --- a/src/system/kernel/timer.cpp +++ b/src/system/kernel/timer.cpp @@ -33,7 +33,7 @@ struct per_cpu_timer_data { bigtime_t real_time_offset; }; -static per_cpu_timer_data sPerCPU[B_MAX_CPU_COUNT]; +static per_cpu_timer_data sPerCPU[SMP_MAX_CPUS]; //#define TRACE_TIMER @@ -383,7 +383,7 @@ cancel_timer(timer* event) int cpu = event->cpu; SpinLocker spinLocker; while (true) { - if (cpu >= B_MAX_CPU_COUNT) + if (cpu >= SMP_MAX_CPUS) return false; spinLocker.SetTo(sPerCPU[cpu].lock, false); From 3106f832a9d4438498de6a606de4c040edb8addb Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 19:57:04 +0100 Subject: [PATCH 182/273] kernel/smp: Fix warning --- src/system/kernel/smp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 9e987a5212..bc7f58323b 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -1366,7 +1366,7 @@ smp_cpu_rendezvous(uint32* var) { atomic_add((int32*)var, 1); - while ((uint32)atomic_get((int32*)var) < sNumCPUs) + while (atomic_get((int32*)var) < sNumCPUs) cpu_wait((int32*)var, sNumCPUs); } From f6b71d5518dab1e63976af2b6518579c121f4186 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 21:05:05 +0100 Subject: [PATCH 183/273] libroot/malloc: Do not use B_MAX_CPU_COUNT --- src/system/libroot/posix/malloc/heap.h | 9 ----- .../libroot/posix/malloc/processheap.cpp | 36 ++++++++++++++----- src/system/libroot/posix/malloc/processheap.h | 20 +++++++---- src/system/libroot/posix/malloc/wrapper.cpp | 2 +- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/src/system/libroot/posix/malloc/heap.h b/src/system/libroot/posix/malloc/heap.h index 9d0c51a6ea..84747c27c5 100644 --- a/src/system/libroot/posix/malloc/heap.h +++ b/src/system/libroot/posix/malloc/heap.h @@ -57,15 +57,6 @@ class hoardHeap { // empty. enum { MAX_EMPTY_SUPERBLOCKS = EMPTY_FRACTION }; - // The maximum number of thread heaps we allow. (NOT the maximum - // number of threads -- Hoard imposes no such limit.) This must be - // a power of two! NB: This number is twice the maximum number of - // PROCESSORS supported by Hoard. - enum { MAX_HEAPS = B_MAX_CPU_COUNT * 2 }; - - // ANDing with this rounds to MAX_HEAPS. - enum { MAX_HEAPS_MASK = MAX_HEAPS - 1 }; - // // The number of size classes. This combined with the // SIZE_CLASS_BASE determine the maximum size of an object. diff --git a/src/system/libroot/posix/malloc/processheap.cpp b/src/system/libroot/posix/malloc/processheap.cpp index 989e145a93..808c42a24f 100644 --- a/src/system/libroot/posix/malloc/processheap.cpp +++ b/src/system/libroot/posix/malloc/processheap.cpp @@ -35,25 +35,45 @@ using namespace BPrivate; -processHeap::processHeap(void) - : _buffer(NULL), _bufferCount(0) +processHeap::processHeap() + : + kMaxThreadHeaps(_numProcessors * 2), + theap((HEAPTYPE*)hoardSbrk(sizeof(HEAPTYPE) * kMaxThreadHeaps)), #if HEAP_FRAG_STATS - , _currentAllocated(0), + _currentAllocated(0), _currentRequested(0), - _maxAllocated(0), _inUseAtMaxAllocated(0), _maxRequested(0) + _maxAllocated(0), + _inUseAtMaxAllocated(0), + _maxRequested(0), #endif +#if HEAP_LOG + _log((Log*) + hoardSbrk(sizeof(Log) * (kMaxThreadHeaps + 1))), +#endif + _buffer(NULL), + _bufferCount(0) { + if (theap == NULL) + return; + new(theap) HEAPTYPE[kMaxThreadHeaps]; + +#if HEAP_LOG + if (_log == NULL) + return; + new(_log) Log[kMaxThreadHeaps + 1]; +#endif + int i; // The process heap is heap 0. setIndex(0); - for (i = 0; i < MAX_HEAPS; i++) { + for (i = 0; i < kMaxThreadHeaps; i++) { // Set every thread's process heap to this one. theap[i].setpHeap(this); // Set every thread heap's index. theap[i].setIndex(i + 1); } #if HEAP_LOG - for (i = 0; i < MAX_HEAPS + 1; i++) { + for (i = 0; i < kMaxThreadHeaps + 1; i++) { char fname[255]; sprintf(fname, "log%d", i); unlink(fname); @@ -74,7 +94,7 @@ processHeap::stats(void) #if HEAP_STATS int umax = 0; int amax = 0; - for (int j = 0; j < MAX_HEAPS; j++) { + for (int j = 0; j < kMaxThreadHeaps; j++) { for (int i = 0; i < SIZE_CLASSES; i++) { amax += theap[j].maxAllocated(i) * sizeFromClass(i); umax += theap[j].maxInUse(i) * sizeFromClass(i); @@ -98,7 +118,7 @@ processHeap::stats(void) #if HEAP_LOG printf("closing logs.\n"); fflush(stdout); - for (int i = 0; i < MAX_HEAPS + 1; i++) { + for (int i = 0; i < kMaxThreadHeaps + 1; i++) { _log[i].close(); } #endif diff --git a/src/system/libroot/posix/malloc/processheap.h b/src/system/libroot/posix/malloc/processheap.h index 5d9125e674..4aa17c7fe9 100644 --- a/src/system/libroot/posix/malloc/processheap.h +++ b/src/system/libroot/posix/malloc/processheap.h @@ -51,7 +51,7 @@ class processHeap : public hoardHeap { // we parcel out. enum { REFILL_NUMBER_OF_SUPERBLOCKS = 16 }; - processHeap(void); + processHeap(); ~processHeap(void) { #if HEAP_STATS @@ -132,8 +132,14 @@ class processHeap : public hoardHeap { processHeap(const processHeap &); const processHeap & operator=(const processHeap &); + // The maximum number of thread heaps we allow. (NOT the maximum + // number of threads -- Hoard imposes no such limit.) This must be + // a power of two! NB: This number is twice the maximum number of + // PROCESSORS supported by Hoard. + const int kMaxThreadHeaps; + // The per-thread heaps. - HEAPTYPE theap[MAX_HEAPS]; + HEAPTYPE* theap; #if HEAP_FRAG_STATS // Statistics required to compute fragmentation. We cannot @@ -152,7 +158,7 @@ class processHeap : public hoardHeap { #endif #if HEAP_LOG - Log < MemoryRequest > _log[MAX_HEAPS + 1]; + Log < MemoryRequest >* _log; #endif // A lock for the superblock buffer. @@ -166,8 +172,9 @@ class processHeap : public hoardHeap { HEAPTYPE & processHeap::getHeap(int i) { + assert(theap != NULL); assert(i >= 0); - assert(i < MAX_HEAPS); + assert(i < kMaxThreadHeaps); return theap[i]; } @@ -176,8 +183,9 @@ processHeap::getHeap(int i) Log & processHeap::getLog(int i) { + assert(_log != NULL); assert(i >= 0); - assert(i < MAX_HEAPS + 1); + assert(i < kMaxThreadHeaps + 1); return _log[i]; } #endif @@ -192,7 +200,7 @@ processHeap::getHeapIndex(void) // In fact, for efficiency, we just round up to the highest power of two, // times two. int tid = find_thread(NULL) & _numProcessorsMask; - assert(tid < MAX_HEAPS); + assert(tid < kMaxThreadHeaps); return tid; } diff --git a/src/system/libroot/posix/malloc/wrapper.cpp b/src/system/libroot/posix/malloc/wrapper.cpp index 294901d13c..5b5e06680e 100644 --- a/src/system/libroot/posix/malloc/wrapper.cpp +++ b/src/system/libroot/posix/malloc/wrapper.cpp @@ -250,7 +250,7 @@ inline static processHeap * getAllocator(void) { static char *buffer = (char *)hoardSbrk(sizeof(processHeap)); - static processHeap *theAllocator = new (buffer) processHeap; + static processHeap *theAllocator = new (buffer) processHeap(); return theAllocator; } From fd96cf08a732197b46f6e71e44c3b4c28d3a28a9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 21:54:07 +0100 Subject: [PATCH 184/273] libroot: Remove _SC_NPROCESSORS_MAX and _SC_CPUID_MAX _SC_NPROCESSORS_MAX and _SC_CPUID_MAX appear to be supported only by Solaris. There isn't much point in exposing such values to the userland anyway. --- headers/posix/unistd.h | 2 -- src/system/libroot/posix/unistd/conf.cpp | 4 ---- 2 files changed, 6 deletions(-) diff --git a/headers/posix/unistd.h b/headers/posix/unistd.h index 4cc5e3f967..cbe72ff4ad 100644 --- a/headers/posix/unistd.h +++ b/headers/posix/unistd.h @@ -104,10 +104,8 @@ /* TODO: check */ #define _SC_IOV_MAX 32 #define _SC_UIO_MAXIOV _SC_IOV_MAX -#define _SC_NPROCESSORS_MAX 33 #define _SC_NPROCESSORS_CONF 34 #define _SC_NPROCESSORS_ONLN 35 -#define _SC_CPUID_MAX 36 #define _SC_ATEXIT_MAX 37 #define _SC_PASS_MAX 39 #define _SC_PHYS_PAGES 40 diff --git a/src/system/libroot/posix/unistd/conf.cpp b/src/system/libroot/posix/unistd/conf.cpp index dfe67688c9..688feac029 100644 --- a/src/system/libroot/posix/unistd/conf.cpp +++ b/src/system/libroot/posix/unistd/conf.cpp @@ -98,8 +98,6 @@ __sysconf(int name) return _POSIX_THREADS; case _SC_IOV_MAX: return IOV_MAX; - case _SC_NPROCESSORS_MAX: - return B_MAX_CPU_COUNT; case _SC_NPROCESSORS_CONF: { system_info info; @@ -125,8 +123,6 @@ __sysconf(int name) count++; return count; } - case _SC_CPUID_MAX: - return B_MAX_CPU_COUNT - 1; case _SC_ATEXIT_MAX: return ATEXIT_MAX; case _SC_PASS_MAX: From bcfdf88186a46a982b684c89dc76b4f6c3d53bf4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 6 Dec 2013 21:57:58 +0100 Subject: [PATCH 185/273] apps, kernel: Remove B_MAX_CPU_COUNT --- headers/os/kernel/OS.h | 19 +--------------- .../processcontroller/ProcessController.cpp | 22 ++++++++++++++++--- .../processcontroller/ProcessController.h | 7 +++--- src/apps/pulse/MiniPulseView.cpp | 2 +- src/apps/pulse/PulseView.cpp | 21 +++++++++++++++--- src/apps/pulse/PulseView.h | 6 +++-- src/system/kernel/arch/x86/arch_smp.cpp | 2 +- 7 files changed, 48 insertions(+), 31 deletions(-) diff --git a/headers/os/kernel/OS.h b/headers/os/kernel/OS.h index 1b2e7f8849..2ba449ce09 100644 --- a/headers/os/kernel/OS.h +++ b/headers/os/kernel/OS.h @@ -420,23 +420,6 @@ extern void ktrace_vprintf(const char *format, va_list args); /* System information */ -#if __INTEL__ -# define B_MAX_CPU_COUNT 8 -#elif __x86_64__ -# define B_MAX_CPU_COUNT 8 -#elif __POWERPC__ -# define B_MAX_CPU_COUNT 8 -#elif __M68K__ -# define B_MAX_CPU_COUNT 1 -#elif __ARM__ -# define B_MAX_CPU_COUNT 1 -#elif __MIPSEL__ -# define B_MAX_CPU_COUNT 1 -#else -# warning Unknown cpu -# define B_MAX_CPU_COUNT 1 -#endif - typedef enum cpu_types { /* TODO: add latest models */ @@ -740,7 +723,7 @@ typedef struct { int32 cpu_count; /* number of cpus */ enum cpu_types cpu_type; /* type of cpu */ int32 cpu_revision; /* revision # of cpu */ - cpu_info cpu_infos[B_MAX_CPU_COUNT]; /* info about individual cpus */ + cpu_info cpu_infos[8]; /* info about individual cpus */ int64 cpu_clock_speed; /* processor clock speed (Hz) */ int64 bus_clock_speed; /* bus clock speed (Hz) */ enum platform_types platform_type; /* type of machine we're on */ diff --git a/src/apps/processcontroller/ProcessController.cpp b/src/apps/processcontroller/ProcessController.cpp index 3987582371..8e9765ea35 100644 --- a/src/apps/processcontroller/ProcessController.cpp +++ b/src/apps/processcontroller/ProcessController.cpp @@ -158,7 +158,11 @@ ProcessController::ProcessController(BRect frame, bool temp) fTrackerIcon(kTrackerSig), fDeskbarIcon(kDeskbarSig), fTerminalIcon(kTerminalSig), - fTemp(temp) + kCPUCount(sysconf(_SC_NPROCESSORS_CONF)), + fTemp(temp), + fLastBarHeight(new float[kCPUCount]), + fCPUTimes(new double[kCPUCount]), + fPrevActive(new bigtime_t[kCPUCount]) { if (!temp) { Init(); @@ -178,7 +182,11 @@ ProcessController::ProcessController(BMessage *data) fTrackerIcon(kTrackerSig), fDeskbarIcon(kDeskbarSig), fTerminalIcon(kTerminalSig), - fTemp(false) + kCPUCount(sysconf(_SC_NPROCESSORS_CONF)), + fTemp(false), + fLastBarHeight(new float[kCPUCount]), + fCPUTimes(new double[kCPUCount]), + fPrevActive(new bigtime_t[kCPUCount]) { Init(); } @@ -191,7 +199,11 @@ ProcessController::ProcessController() fTrackerIcon(kTrackerSig), fDeskbarIcon(kDeskbarSig), fTerminalIcon(kTerminalSig), - fTemp(false) + kCPUCount(sysconf(_SC_NPROCESSORS_CONF)), + fTemp(false), + fLastBarHeight(new float[kCPUCount]), + fCPUTimes(new double[kCPUCount]), + fPrevActive(new bigtime_t[kCPUCount]) { Init(); } @@ -208,6 +220,10 @@ ProcessController::~ProcessController() delete fMessageRunner; gPCView = NULL; + + delete[] fPrevActive; + delete[] fCPUTimes; + delete[] fLastBarHeight; } diff --git a/src/apps/processcontroller/ProcessController.h b/src/apps/processcontroller/ProcessController.h index 02cd76f067..80c3a08c2b 100644 --- a/src/apps/processcontroller/ProcessController.h +++ b/src/apps/processcontroller/ProcessController.h @@ -60,12 +60,13 @@ class ProcessController : public BView { void Init(); void _HandleDebugRequest(team_id team, thread_id thread); + const int32 kCPUCount; bool fTemp; float fMemoryUsage; - float fLastBarHeight[B_MAX_CPU_COUNT]; + float* fLastBarHeight; float fLastMemoryHeight; - double fCPUTimes[B_MAX_CPU_COUNT]; - bigtime_t fPrevActive[B_MAX_CPU_COUNT]; + double* fCPUTimes; + bigtime_t* fPrevActive; bigtime_t fPrevTime; BMessageRunner* fMessageRunner; rgb_color frame_color, active_color, idle_color, memory_color, swap_color; diff --git a/src/apps/pulse/MiniPulseView.cpp b/src/apps/pulse/MiniPulseView.cpp index e36089869c..923810f439 100644 --- a/src/apps/pulse/MiniPulseView.cpp +++ b/src/apps/pulse/MiniPulseView.cpp @@ -64,7 +64,7 @@ MiniPulseView::MiniPulseView(BMessage *message) void MiniPulseView::Draw(BRect rect) { system_info sys_info; get_system_info(&sys_info); - if (sys_info.cpu_count > B_MAX_CPU_COUNT || sys_info.cpu_count <= 0) + if (sys_info.cpu_count <= 0) return; BRect bounds(Bounds()); diff --git a/src/apps/pulse/PulseView.cpp b/src/apps/pulse/PulseView.cpp index bc5969a0af..9370875a59 100644 --- a/src/apps/pulse/PulseView.cpp +++ b/src/apps/pulse/PulseView.cpp @@ -26,8 +26,14 @@ #define B_TRANSLATION_CONTEXT "PulseView" -PulseView::PulseView(BRect rect, const char *name) : - BView(rect, name, B_FOLLOW_ALL_SIDES, B_WILL_DRAW | B_PULSE_NEEDED | B_FRAME_EVENTS) { +PulseView::PulseView(BRect rect, const char *name) + : + BView(rect, name, B_FOLLOW_ALL_SIDES, + B_WILL_DRAW | B_PULSE_NEEDED | B_FRAME_EVENTS), + kCPUCount(sysconf(_SC_NPROCESSORS_CONF)), + cpu_times(new double[kCPUCount]), + prev_active(new bigtime_t[kCPUCount]) +{ popupmenu = NULL; cpu_menu_items = NULL; @@ -40,7 +46,13 @@ PulseView::PulseView(BRect rect, const char *name) : } // This version will be used by the instantiated replicant -PulseView::PulseView(BMessage *message) : BView(message) { +PulseView::PulseView(BMessage *message) + : + BView(message), + kCPUCount(sysconf(_SC_NPROCESSORS_CONF)), + cpu_times(new double[kCPUCount]), + prev_active(new bigtime_t[kCPUCount]) +{ SetResizingMode(B_FOLLOW_ALL_SIDES); SetFlags(B_WILL_DRAW | B_PULSE_NEEDED); @@ -138,5 +150,8 @@ void PulseView::ChangeCPUState(BMessage *message) { PulseView::~PulseView() { if (popupmenu != NULL) delete popupmenu; if (cpu_menu_items != NULL) delete cpu_menu_items; + + delete[] prev_active; + delete[] cpu_times; } diff --git a/src/apps/pulse/PulseView.h b/src/apps/pulse/PulseView.h index b61922a886..d6e0f9f4d6 100644 --- a/src/apps/pulse/PulseView.h +++ b/src/apps/pulse/PulseView.h @@ -32,8 +32,10 @@ class PulseView : public BView { BMenuItem *mode1, *mode2, *preferences, *about; BMenuItem **cpu_menu_items; - double cpu_times[B_MAX_CPU_COUNT]; - bigtime_t prev_active[B_MAX_CPU_COUNT]; + const int32 kCPUCount; + + double* cpu_times; + bigtime_t* prev_active; bigtime_t prev_time; }; diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index 0a97a97e51..adf89a247c 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -73,7 +73,7 @@ x86_smp_error_interrupt(void *data) uint32 x86_get_cpu_apic_id(int32 cpu) { - ASSERT(cpu >= 0 && cpu < B_MAX_CPU_COUNT); + ASSERT(cpu >= 0 && cpu < SMP_MAX_CPUS); return sCPUAPICIds[cpu]; } From 02749e2429d9ccc09f94febaf316da7803ecb922 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 11 Dec 2013 01:59:31 +0100 Subject: [PATCH 186/273] x86/paging: Do not use array placement new Array placement new has implementation defined memory overhead what makes it pretty useless. --- .../kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp | 6 +++--- .../kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp index e8e0419b07..64c93cf8be 100644 --- a/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp +++ b/src/system/kernel/arch/x86/paging/32bit/X86PagingMethod32Bit.cpp @@ -295,10 +295,10 @@ X86PagingMethod32Bit::Init(kernel_args* args, // create the initial pools for the physical page mapper int32 poolCount = _GetInitialPoolCount(); - PhysicalPageSlotPool* pool - = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) - PhysicalPageSlotPool[poolCount]; + PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool; + for (int32 i = 0; i < poolCount; i++) { + new(&pool[i]) PhysicalPageSlotPool; status_t error = pool[i].InitInitial(args); if (error != B_OK) { panic("X86PagingMethod32Bit::Init(): Failed to create initial pool " diff --git a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp index 38d4e0e676..089f0ccb84 100644 --- a/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp +++ b/src/system/kernel/arch/x86/paging/pae/X86PagingMethodPAE.cpp @@ -595,10 +595,10 @@ X86PagingMethodPAE::Init(kernel_args* args, // create the initial pools for the physical page mapper int32 poolCount = _GetInitialPoolCount(); - PhysicalPageSlotPool* pool - = new(&PhysicalPageSlotPool::sInitialPhysicalPagePool) - PhysicalPageSlotPool[poolCount]; + PhysicalPageSlotPool* pool = PhysicalPageSlotPool::sInitialPhysicalPagePool; + for (int32 i = 0; i < poolCount; i++) { + new(&pool[i]) PhysicalPageSlotPool; status_t error = pool[i].InitInitial(this, args); if (error != B_OK) { panic("X86PagingMethodPAE::Init(): Failed to create initial pool " From fa157509258c9b494903e48aed66816897bb6cc6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 11 Dec 2013 04:36:49 +0100 Subject: [PATCH 187/273] kernel/cpu: Do not use get_system_info() to identify CPU --- src/add-ons/kernel/cpu/x86/amd.cpp | 10 +++------- src/add-ons/kernel/cpu/x86/intel.cpp | 8 ++------ src/add-ons/kernel/cpu/x86/via.cpp | 8 ++------ 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/add-ons/kernel/cpu/x86/amd.cpp b/src/add-ons/kernel/cpu/x86/amd.cpp index 5d95c01c9a..d3fd31aaad 100644 --- a/src/add-ons/kernel/cpu/x86/amd.cpp +++ b/src/add-ons/kernel/cpu/x86/amd.cpp @@ -10,7 +10,7 @@ #include "amd.h" #include "generic_x86.h" -#include +#include static void @@ -23,16 +23,12 @@ amd_init_mtrrs(void) static status_t amd_init(void) { - system_info info; - if (get_system_info(&info) != B_OK) - return B_ERROR; - - if ((info.cpu_type & B_CPU_x86_VENDOR_MASK) != B_CPU_AMD_x86) + if (gCPU[0].arch.vendor != VENDOR_AMD) return B_ERROR; // The K6-2 doesn't seem to support write-combining (before model 9), // so we ignore anything before that one. - if (info.cpu_type <= B_CPU_AMD_K6_2) + if (gCPU[0].arch.family <= 5 || gCPU[0].arch.model < 9) return B_ERROR; generic_mtrr_compute_physical_mask(); diff --git a/src/add-ons/kernel/cpu/x86/intel.cpp b/src/add-ons/kernel/cpu/x86/intel.cpp index a6935eb0ed..1bb9e72d53 100644 --- a/src/add-ons/kernel/cpu/x86/intel.cpp +++ b/src/add-ons/kernel/cpu/x86/intel.cpp @@ -10,7 +10,7 @@ #include "intel.h" #include "generic_x86.h" -#include +#include static void @@ -23,11 +23,7 @@ intel_init_mtrrs(void) static status_t intel_init(void) { - system_info info; - if (get_system_info(&info) != B_OK) - return B_ERROR; - - if ((info.cpu_type & B_CPU_x86_VENDOR_MASK) != B_CPU_INTEL_x86) + if (gCPU[0].arch.vendor != VENDOR_INTEL) return B_ERROR; generic_mtrr_compute_physical_mask(); diff --git a/src/add-ons/kernel/cpu/x86/via.cpp b/src/add-ons/kernel/cpu/x86/via.cpp index 6a222ff24d..4924886156 100644 --- a/src/add-ons/kernel/cpu/x86/via.cpp +++ b/src/add-ons/kernel/cpu/x86/via.cpp @@ -10,7 +10,7 @@ #include "via.h" #include "generic_x86.h" -#include +#include static uint32 @@ -41,11 +41,7 @@ via_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count) static status_t via_init(void) { - system_info info; - if (get_system_info(&info) != B_OK) - return B_ERROR; - - if ((info.cpu_type & B_CPU_x86_VENDOR_MASK) != B_CPU_VIA_IDT_x86) + if (gCPU[0].arch.vendor != VENDOR_CENTAUR) return B_ERROR; // current VIA CPUs have always 36 bit (or less?) From 4c125ccacd874f5c68851184e2d6f965087b5df0 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 12 Dec 2013 03:00:18 +0100 Subject: [PATCH 188/273] kits/media: CPU clock speed may vary significantly --- src/kits/media/MediaRoster.cpp | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/kits/media/MediaRoster.cpp b/src/kits/media/MediaRoster.cpp index 5c0721e40b..78f8d1ce28 100644 --- a/src/kits/media/MediaRoster.cpp +++ b/src/kits/media/MediaRoster.cpp @@ -3112,25 +3112,10 @@ BMediaRoster::AudioBufferSizeFor(int32 channelCount, uint32 sampleFormat, bigtime_t bufferDuration; ssize_t bufferSize; - system_info info; - get_system_info(&info); - - if (info.cpu_clock_speed > 2000000000) // 2 GHz - bufferDuration = 2500; - else if (info.cpu_clock_speed > 1000000000) - bufferDuration = 5000; - else if (info.cpu_clock_speed > 600000000) - bufferDuration = 10000; - else if (info.cpu_clock_speed > 200000000) - bufferDuration = 20000; - else if (info.cpu_clock_speed > 100000000) - bufferDuration = 30000; - else - bufferDuration = 50000; - - if ((busKind == B_ISA_BUS || busKind == B_PCMCIA_BUS) - && bufferDuration < 25000) + if (busKind == B_ISA_BUS || busKind == B_PCMCIA_BUS) bufferDuration = 25000; + else + bufferDuration = 10000; bufferSize = (sampleFormat & 0xf) * channelCount * (ssize_t)((frameRate * bufferDuration) / 1000000.0); From 39f6ced055ffa25d6a763dbf8be54cd5b1d1a3f1 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 12 Dec 2013 03:18:47 +0100 Subject: [PATCH 189/273] bin/top: Remove busy_wait_time --- src/bin/top.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/bin/top.c b/src/bin/top.c index ad92f0553a..6ee6a1022d 100644 --- a/src/bin/top.c +++ b/src/bin/top.c @@ -151,8 +151,6 @@ static void compare( thread_time_list_t *old, thread_time_list_t *new, - bigtime_t old_busy, - bigtime_t new_busy, bigtime_t uinterval, int refresh ) @@ -238,7 +236,7 @@ compare( printf("%6s %7s %7s %7s %4s %16s %16s\n", "thid", "total", "user", "kernel", "%cpu", "team name", "thread name"); linecount = 1; - idletime = new_busy - old_busy; + idletime = 0; gtotal = 0; ktotal = 0; utotal = 0; @@ -338,7 +336,6 @@ setup_term(bool onlyRows) static thread_time_list_t gather( thread_time_list_t *old, - bigtime_t *busy_wait_time, int refresh ) { @@ -347,7 +344,6 @@ gather( thread_info t; team_info tm; thread_time_list_t times; - bigtime_t old_busy; int i; system_info info; bigtime_t oldLastMeasure; @@ -358,10 +354,6 @@ gather( oldLastMeasure = lastMeasure; lastMeasure = system_time(); - get_system_info(&info); - old_busy = *busy_wait_time; - *busy_wait_time = info._busy_wait_time; - while (get_next_team_info(&tmcookie, &tm) == B_NO_ERROR) { thcookie = 0; while (get_next_thread_info(tm.team, &thcookie, &t) == B_NO_ERROR) { @@ -377,8 +369,7 @@ gather( setup_term(true); screen_size_changed = 0; } - compare(old, ×, old_busy, *busy_wait_time, - system_time() - oldLastMeasure, refresh); + compare(old, ×, system_time() - oldLastMeasure, refresh); free_times(old); } return (times); @@ -414,7 +405,6 @@ main(int argc, char **argv) bigtime_t then; bigtime_t uinterval; bigtime_t elapsed; - bigtime_t busy; char *myname; get_system_info (&sysinfo); @@ -460,22 +450,22 @@ main(int argc, char **argv) if (iters < 0) { // You will only have to wait half a second for the first iteration. uinterval = 1 * 1000000 / 2; - baseline = gather(NULL, &busy, refresh); + baseline = gather(NULL, refresh); elapsed = system_time() - lastMeasure; if (elapsed < uinterval) snooze(uinterval - elapsed); then = system_time(); - baseline = gather(&baseline, &busy, refresh); + baseline = gather(&baseline, refresh); } else - baseline = gather(NULL, &busy, refresh); + baseline = gather(NULL, refresh); uinterval = interval * 1000000; for (i = 0; iters < 0 || i < iters; i++) { elapsed = system_time() - lastMeasure; if (elapsed < uinterval) snooze(uinterval - elapsed); - baseline = gather(&baseline, &busy, refresh); + baseline = gather(&baseline, refresh); } exit(0); } From 53aac05ee5fcc4b30ab7f2a71c1229c2f23ee321 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 12 Dec 2013 03:56:33 +0100 Subject: [PATCH 190/273] libroot: Add prototypes for __{get,set}_scheduler_mode --- src/system/libroot/os/scheduler.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/system/libroot/os/scheduler.c b/src/system/libroot/os/scheduler.c index 9bae2228b1..4dc16b6225 100644 --- a/src/system/libroot/os/scheduler.c +++ b/src/system/libroot/os/scheduler.c @@ -32,6 +32,9 @@ static struct { {(uint32)-1, -1} }; +status_t __set_scheduler_mode(int32 mode); +int32 __get_scheduler_mode(void); + int32 suggest_thread_priority(uint32 what, int32 period, bigtime_t jitter, From 82727571c323b109214bb78496740b9b4b861a12 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 12 Dec 2013 04:00:56 +0100 Subject: [PATCH 191/273] libroot: Fix uninitialized counter in mutex_lock() --- src/system/libroot/os/locks/mutex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/libroot/os/locks/mutex.cpp b/src/system/libroot/os/locks/mutex.cpp index 7c017e73ff..93fca56d5f 100644 --- a/src/system/libroot/os/locks/mutex.cpp +++ b/src/system/libroot/os/locks/mutex.cpp @@ -59,7 +59,7 @@ mutex_destroy(mutex *lock) status_t mutex_lock(mutex *lock) { - uint32 count; + uint32 count = 0; const uint32 kMaxCount = (lock->flags & MUTEX_FLAG_ADAPTIVE) != 0 ? MAX_UNSUCCESSFUL_SPINS : 1; From 8cf2f36c3740e02e49c0a5aace44418e070c225e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 12 Dec 2013 21:14:25 +0100 Subject: [PATCH 192/273] libroot: Fix Hoard when the CPU count is not a power of two --- src/system/libroot/posix/malloc/heap.cpp | 5 +++-- src/system/libroot/posix/malloc/heap.h | 6 ++++++ src/system/libroot/posix/malloc/processheap.cpp | 17 ++++++++--------- src/system/libroot/posix/malloc/processheap.h | 12 +++--------- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/system/libroot/posix/malloc/heap.cpp b/src/system/libroot/posix/malloc/heap.cpp index 5ded94723e..99aba542d2 100644 --- a/src/system/libroot/posix/malloc/heap.cpp +++ b/src/system/libroot/posix/malloc/heap.cpp @@ -99,6 +99,7 @@ size_t hoardHeap::_threshold[hoardHeap::SIZE_CLASSES] = { #endif +int hoardHeap::fMaxThreadHeaps = 1; int hoardHeap::_numProcessors; int hoardHeap::_numProcessorsMask; @@ -454,7 +455,7 @@ hoardHeap::initNumProcs(void) else hoardHeap::_numProcessors = info.cpu_count; - hoardHeap::_numProcessorsMask = - (1 << (lg(hoardHeap::_numProcessors) + 1)) - 1; + fMaxThreadHeaps = 1 << (lg(_numProcessors) + 1); + _numProcessorsMask = fMaxThreadHeaps - 1; } diff --git a/src/system/libroot/posix/malloc/heap.h b/src/system/libroot/posix/malloc/heap.h index 84747c27c5..b2a9ed6ba3 100644 --- a/src/system/libroot/posix/malloc/heap.h +++ b/src/system/libroot/posix/malloc/heap.h @@ -205,6 +205,12 @@ class hoardHeap { static void initNumProcs(void); protected: + // The maximum number of thread heaps we allow. (NOT the maximum + // number of threads -- Hoard imposes no such limit.) This must be + // a power of two! NB: This number is twice the maximum number of + // PROCESSORS supported by Hoard. + static int fMaxThreadHeaps; + // number of CPUs, cached static int _numProcessors; static int _numProcessorsMask; diff --git a/src/system/libroot/posix/malloc/processheap.cpp b/src/system/libroot/posix/malloc/processheap.cpp index 808c42a24f..129cf4c3a0 100644 --- a/src/system/libroot/posix/malloc/processheap.cpp +++ b/src/system/libroot/posix/malloc/processheap.cpp @@ -37,8 +37,7 @@ using namespace BPrivate; processHeap::processHeap() : - kMaxThreadHeaps(_numProcessors * 2), - theap((HEAPTYPE*)hoardSbrk(sizeof(HEAPTYPE) * kMaxThreadHeaps)), + theap((HEAPTYPE*)hoardSbrk(sizeof(HEAPTYPE) * fMaxThreadHeaps)), #if HEAP_FRAG_STATS _currentAllocated(0), _currentRequested(0), @@ -48,32 +47,32 @@ processHeap::processHeap() #endif #if HEAP_LOG _log((Log*) - hoardSbrk(sizeof(Log) * (kMaxThreadHeaps + 1))), + hoardSbrk(sizeof(Log) * (fMaxThreadHeaps + 1))), #endif _buffer(NULL), _bufferCount(0) { if (theap == NULL) return; - new(theap) HEAPTYPE[kMaxThreadHeaps]; + new(theap) HEAPTYPE[fMaxThreadHeaps]; #if HEAP_LOG if (_log == NULL) return; - new(_log) Log[kMaxThreadHeaps + 1]; + new(_log) Log[fMaxThreadHeaps + 1]; #endif int i; // The process heap is heap 0. setIndex(0); - for (i = 0; i < kMaxThreadHeaps; i++) { + for (i = 0; i < fMaxThreadHeaps; i++) { // Set every thread's process heap to this one. theap[i].setpHeap(this); // Set every thread heap's index. theap[i].setIndex(i + 1); } #if HEAP_LOG - for (i = 0; i < kMaxThreadHeaps + 1; i++) { + for (i = 0; i < fMaxThreadHeaps + 1; i++) { char fname[255]; sprintf(fname, "log%d", i); unlink(fname); @@ -94,7 +93,7 @@ processHeap::stats(void) #if HEAP_STATS int umax = 0; int amax = 0; - for (int j = 0; j < kMaxThreadHeaps; j++) { + for (int j = 0; j < fMaxThreadHeaps; j++) { for (int i = 0; i < SIZE_CLASSES; i++) { amax += theap[j].maxAllocated(i) * sizeFromClass(i); umax += theap[j].maxInUse(i) * sizeFromClass(i); @@ -118,7 +117,7 @@ processHeap::stats(void) #if HEAP_LOG printf("closing logs.\n"); fflush(stdout); - for (int i = 0; i < kMaxThreadHeaps + 1; i++) { + for (int i = 0; i < fMaxThreadHeaps + 1; i++) { _log[i].close(); } #endif diff --git a/src/system/libroot/posix/malloc/processheap.h b/src/system/libroot/posix/malloc/processheap.h index 4aa17c7fe9..f2222897f2 100644 --- a/src/system/libroot/posix/malloc/processheap.h +++ b/src/system/libroot/posix/malloc/processheap.h @@ -132,12 +132,6 @@ class processHeap : public hoardHeap { processHeap(const processHeap &); const processHeap & operator=(const processHeap &); - // The maximum number of thread heaps we allow. (NOT the maximum - // number of threads -- Hoard imposes no such limit.) This must be - // a power of two! NB: This number is twice the maximum number of - // PROCESSORS supported by Hoard. - const int kMaxThreadHeaps; - // The per-thread heaps. HEAPTYPE* theap; @@ -174,7 +168,7 @@ processHeap::getHeap(int i) { assert(theap != NULL); assert(i >= 0); - assert(i < kMaxThreadHeaps); + assert(i < fMaxThreadHeaps); return theap[i]; } @@ -185,7 +179,7 @@ processHeap::getLog(int i) { assert(_log != NULL); assert(i >= 0); - assert(i < kMaxThreadHeaps + 1); + assert(i < fMaxThreadHeaps + 1); return _log[i]; } #endif @@ -200,7 +194,7 @@ processHeap::getHeapIndex(void) // In fact, for efficiency, we just round up to the highest power of two, // times two. int tid = find_thread(NULL) & _numProcessorsMask; - assert(tid < kMaxThreadHeaps); + assert(tid < fMaxThreadHeaps); return tid; } From 25feff2c29fddb8ba75323b2af9b2d7011fa78e7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 12 Dec 2013 21:46:48 +0100 Subject: [PATCH 193/273] ProcessController: Explicitly check thread priority --- src/apps/processcontroller/ThreadBarMenuItem.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/apps/processcontroller/ThreadBarMenuItem.cpp b/src/apps/processcontroller/ThreadBarMenuItem.cpp index 8af186b7ad..94b2d1dd8c 100644 --- a/src/apps/processcontroller/ThreadBarMenuItem.cpp +++ b/src/apps/processcontroller/ThreadBarMenuItem.cpp @@ -104,10 +104,16 @@ ThreadBarMenuItem::DrawBar(bool force) r.right = fGrenze1; } if (r.left < r.right) { - if (selected) - menu->SetHighColor(fThreadID <= gCPUcount ? gIdleColorSelected : gUserColorSelected); - else - menu->SetHighColor(fThreadID <= gCPUcount ? gIdleColor : gUserColor); + thread_info threadInfo; + bool idleThread = false; + if (get_thread_info(fThreadID, &threadInfo) == B_OK) + idleThread = threadInfo.priority == B_IDLE_PRIORITY; + + if (selected) { + menu->SetHighColor( + idleThread ? gIdleColorSelected : gUserColorSelected); + } else + menu->SetHighColor(idleThread ? gIdleColor : gUserColor); menu->FillRect(r); } r.left = grenze2; @@ -157,7 +163,7 @@ ThreadBarMenuItem::BarUpdate() bigtime_t now = system_time(); fKernel = double(info.kernel_time - fThreadInfo.kernel_time) / double(now - fLastTime); fUser = double(info.user_time - fThreadInfo.user_time) / double(now - fLastTime); - if (fThreadID <= gCPUcount) { + if (info.priority == B_IDLE_PRIORITY) { fUser += fKernel; fKernel = 0; } From e397276460e641d58c709d9125ed9b7f4b34433a Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 13 Dec 2013 02:38:35 +0100 Subject: [PATCH 194/273] apps/BSnow: Do not use cpu_clock_speed --- src/apps/bsnow/SnowView.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/bsnow/SnowView.cpp b/src/apps/bsnow/SnowView.cpp index 1a3765d9c7..5c3935a236 100644 --- a/src/apps/bsnow/SnowView.cpp +++ b/src/apps/bsnow/SnowView.cpp @@ -87,7 +87,7 @@ SnowView::SnowView(BMessage *archive) fShowClickMe = false; SetFlags(Flags() & ~B_PULSE_NEEDED); /* it's only used when in the app */ get_system_info(&si); - fNumFlakes = ((int32)(si.cpu_clock_speed/1000000)) * si.cpu_count / 3; //; + fNumFlakes = 1000 * si.cpu_count / 3; printf("BSnow: using %ld flakes\n", fNumFlakes); for (int i = 0; i < WORKSPACES_COUNT; i++) { fFlakes[i] = new flake[fNumFlakes]; From b2504f3b56c55b16e7a911e438083d37d1c81302 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 13 Dec 2013 02:44:29 +0100 Subject: [PATCH 195/273] drivers/graphics: Remove log_{coll, dump} log_coll.c and log_dump.{c, h} don't appear to be used by anything. --- .../kernel/drivers/graphics/common/Jamfile | 2 - .../kernel/drivers/graphics/common/log_coll.c | 129 ------------------ .../kernel/drivers/graphics/common/log_dump.c | 73 ---------- .../kernel/drivers/graphics/common/log_dump.h | 17 --- 4 files changed, 221 deletions(-) delete mode 100644 src/add-ons/kernel/drivers/graphics/common/log_coll.c delete mode 100644 src/add-ons/kernel/drivers/graphics/common/log_dump.c delete mode 100644 src/add-ons/kernel/drivers/graphics/common/log_dump.h diff --git a/src/add-ons/kernel/drivers/graphics/common/Jamfile b/src/add-ons/kernel/drivers/graphics/common/Jamfile index 7ea40920b7..6fb4bde749 100644 --- a/src/add-ons/kernel/drivers/graphics/common/Jamfile +++ b/src/add-ons/kernel/drivers/graphics/common/Jamfile @@ -9,7 +9,5 @@ UsePrivateHeaders [ FDirName graphics radeon ] ; UsePrivateHeaders [ FDirName graphics common ] ; StaticLibrary libgraphicscommon.a : - log_coll.c - log_dump.c memory_manager.c ; diff --git a/src/add-ons/kernel/drivers/graphics/common/log_coll.c b/src/add-ons/kernel/drivers/graphics/common/log_coll.c deleted file mode 100644 index f9dec1e79f..0000000000 --- a/src/add-ons/kernel/drivers/graphics/common/log_coll.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - Copyright (c) 2002, Thomas Kurschel - - Part of Radeon driver - - Both kernel and user space part. - (init and clean-up must be done in - kernel space). -*/ - - -#include "log_coll.h" - -#include -#include - -#include -#include -#include - -typedef struct log_info_t { - char *log_buffer; - uint32 log_buffer_len; - uint32 log_buffer_pos; - area_id area; -} log_info; - -#ifdef ENABLE_LOGGING - - -// write one log entry -void log( log_info *li, uint16 what, const uint8 num_args, ... ) -{ - uint32 pos; - va_list vl; - log_entry *entry; - uint32 i; - uint32 entry_size; - - entry_size = sizeof( log_entry ) + (num_args - 1) * sizeof( uint32 ); - pos = atomic_add( &li->log_buffer_pos, entry_size ); - - if( li->log_buffer_pos > li->log_buffer_len ) { - atomic_add( &li->log_buffer_pos, -entry_size ); - return; - } - - entry = (log_entry *)&li->log_buffer[pos]; - - entry->tsc = system_time(); - entry->what = what; - entry->num_args = num_args; - - va_start( vl, num_args ); - for( i = 0; i < num_args; ++i ) { - entry->args[i] = va_arg( vl, uint32 ); - } - va_end( vl ); -} - -#ifdef LOG_INCLUDE_STARTUP - -// create log buffer -log_info *log_init( uint32 size ) -{ - log_info *li; - area_id area; - - // buffer must be accessible from user mem - // to allow logging from there as well; - // you cannot clone this area as there are - // pointers which would break (it wouldn't be - // hard to get rid of them, but I don't care - // and keep it as simple as possible) - area = create_area( "fast_logger", - (void **)&li, B_ANY_KERNEL_ADDRESS, - (sizeof( log_info ) + size + (B_PAGE_SIZE - 1)) & ~(B_PAGE_SIZE - 1), - B_FULL_LOCK, B_READ_AREA | B_WRITE_AREA ); - - if( area < 0 ) - panic( "Radeon Fast logger: cannot allocate %ld byte for logging data\n", size ); - - li->area = area; - li->log_buffer = (char *)li + sizeof( log_info ); - li->log_buffer_len = size; - li->log_buffer_pos = 0; - - return li; -} - -// clean-up logging -void log_exit( log_info *li ) -{ - li->log_buffer_pos = 0; - //free( li->log_buffer ); - delete_area( li->area ); -} - -#endif - -#endif - - -#ifdef LOG_INCLUDE_STARTUP - -// get *current* size of logging data -uint32 log_getsize( log_info *li ) -{ - if( li == NULL ) - return 0; - - dprintf( "RADEON -- log_getsize: log_pos %ld\n", li->log_buffer_pos ); - return li->log_buffer_pos; -} - -// get up to max_size bytes of logging data -void log_getcopy( log_info *li, void *dest, uint32 max_size ) -{ - if( li == NULL ) - return; - - dprintf( "RADEON -- log_getcopy: max_size %ld, log_pos %ld\n", - max_size, li->log_buffer_pos ); - memcpy( dest, li->log_buffer, min( li->log_buffer_pos, max_size )); - - li->log_buffer_pos = 0; -} - -#endif diff --git a/src/add-ons/kernel/drivers/graphics/common/log_dump.c b/src/add-ons/kernel/drivers/graphics/common/log_dump.c deleted file mode 100644 index fb4d79b8d9..0000000000 --- a/src/add-ons/kernel/drivers/graphics/common/log_dump.c +++ /dev/null @@ -1,73 +0,0 @@ -/* - Copyright (c) 2002, Thomas Kurschel - - - Part of Radeon driver - - Fast logger - functions to create dump -*/ - - -#include -#include - -#include "log_coll.h" -#include "log_dump.h" -#include "log_enum.h" -#include "log_names.h" - -system_info sysinfo; - -// dump one entry -static void log_printentry( FILE *logfile, log_entry *entry ) -{ - uint64 time; - uint32 min, sec, mill, mic; - - time = entry->tsc / (sysinfo.cpu_clock_speed / 1000000); - mic = time % 1000; - time /= 1000; - mill = time % 1000; - time /= 1000; - sec = time % 60; - time /= 60; - min = time; - - fprintf( logfile, "%03ld:%02ld:%03ld.%03ld ", min, sec, mill, mic ); - if( entry->what < sizeof( log_names ) / sizeof( log_names[0] ) ) - fprintf( logfile, log_names[entry->what] ); - else - fprintf( logfile, "unknown %ld", (uint32)entry->what ); - - if( entry->num_args > 0 ) { - uint32 i; - - fprintf( logfile, " (" ); - for( i = 0; i < entry->num_args; ++i ) { - if( i > 0 ) - fprintf( logfile, ", " ); - - fprintf( logfile, "0x%08lx", entry->args[i] ); - } - fprintf( logfile, ")" ); - } - - fprintf( logfile, "\n" ); -} - - -// dump entire log -void log_printall( FILE *logfile, char *buffer, uint32 buffer_len ) -{ - uint32 pos; - - get_system_info( &sysinfo ); - - for( pos = 0; pos < buffer_len; ) { - log_entry *entry; - - entry = (log_entry *)(buffer + pos); - log_printentry( logfile, entry/*, &tsc*/ ); - pos += sizeof( log_entry ) + (entry->num_args - 1) * sizeof( uint32 ); - } -} diff --git a/src/add-ons/kernel/drivers/graphics/common/log_dump.h b/src/add-ons/kernel/drivers/graphics/common/log_dump.h deleted file mode 100644 index bb2a775b91..0000000000 --- a/src/add-ons/kernel/drivers/graphics/common/log_dump.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - Copyright (c) 2002, Thomas Kurschel - - - Part of Radeon accelerant - - Fast logger - functions to create dump -*/ - -#ifndef __LOG_DUMP_H__ -#define __LOG_DUMP_H__ - -#include - -void log_printall( FILE *logfile, char *buffer, uint32 buffer_len ); - -#endif From 1bc7045fdfb85e6151d01c73669be19627c4783b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 16 Dec 2013 03:58:43 +0100 Subject: [PATCH 196/273] kernel, libroot: Introduce new API for obtaining system info --- headers/os/kernel/OS.h | 358 +++--------- headers/private/kernel/arch/system_info.h | 2 +- headers/private/kernel/ksystem_info.h | 6 +- headers/private/shared/cpu_type.h | 527 +++++++++--------- headers/private/system/syscalls.h | 8 +- headers/private/system/system_info.h | 4 + src/apps/aboutsystem/AboutSystem.cpp | 35 +- src/apps/activitymonitor/DataSource.cpp | 6 +- src/apps/activitymonitor/SystemInfo.cpp | 3 + src/apps/activitymonitor/SystemInfo.h | 3 + .../controllers/DebugReportGenerator.cpp | 34 +- src/apps/diskprobe/TypeEditors.cpp | 2 +- src/apps/packageinstaller/PackageInfo.cpp | 33 +- src/apps/processcontroller/MemoryBarMenu.cpp | 4 +- src/apps/processcontroller/MemoryBarMenu.h | 2 +- .../processcontroller/ProcessController.cpp | 28 +- .../processcontroller/ProcessController.h | 2 +- .../processcontroller/TeamBarMenuItem.cpp | 5 +- src/apps/pulse/DeskbarPulseView.cpp | 3 +- src/apps/pulse/MiniPulseView.cpp | 5 +- src/apps/pulse/NormalPulseView.cpp | 43 +- src/apps/pulse/PulseApp.cpp | 4 +- src/apps/pulse/PulseApp.h | 2 +- src/apps/pulse/PulseView.cpp | 14 +- src/bin/Jamfile | 2 +- src/bin/coreutils/src/uname.c | 24 +- src/bin/sysinfo.cpp | 114 ++-- src/kits/tracker/TaskLoop.cpp | 10 +- src/servers/app/drawing/Painter/Painter.cpp | 2 +- .../kernel/arch/x86/arch_system_info.cpp | 130 ++--- src/system/kernel/system_info.cpp | 164 +++++- src/system/libroot/os/Jamfile | 2 +- src/system/libroot/os/system_info.c | 59 -- src/system/libroot/os/system_info.cpp | 242 ++++++++ src/system/libroot/posix/sys/uname.c | 33 +- 35 files changed, 1107 insertions(+), 808 deletions(-) delete mode 100644 src/system/libroot/os/system_info.c create mode 100644 src/system/libroot/os/system_info.cpp diff --git a/headers/os/kernel/OS.h b/headers/os/kernel/OS.h index 2ba449ce09..d6199a3d6a 100644 --- a/headers/os/kernel/OS.h +++ b/headers/os/kernel/OS.h @@ -420,220 +420,108 @@ extern void ktrace_vprintf(const char *format, va_list args); /* System information */ -typedef enum cpu_types { - /* TODO: add latest models */ +typedef struct { + bigtime_t active_time; /* usec of doing useful work since boot */ + int32 load; + bool enabled; +} cpu_info; - /* Motorola/IBM */ - B_CPU_PPC_UNKNOWN = 0, - B_CPU_PPC_601 = 1, - B_CPU_PPC_602 = 7, - B_CPU_PPC_603 = 2, - B_CPU_PPC_603e = 3, - B_CPU_PPC_603ev = 8, - B_CPU_PPC_604 = 4, - B_CPU_PPC_604e = 5, - B_CPU_PPC_604ev = 9, - B_CPU_PPC_620 = 10, - B_CPU_PPC_750 = 6, - B_CPU_PPC_686 = 13, - B_CPU_PPC_860 = 25, - B_CPU_PPC_7400 = 26, - B_CPU_PPC_7410 = 27, - B_CPU_PPC_7447A = 28, - B_CPU_PPC_7448 = 29, - B_CPU_PPC_7450 = 30, - B_CPU_PPC_7455 = 31, - B_CPU_PPC_7457 = 32, - B_CPU_PPC_8240 = 33, - B_CPU_PPC_8245 = 34, +typedef struct { + bigtime_t boot_time; /* time of boot (usecs since 1/1/1970) */ - B_CPU_PPC_IBM_401A1 = 35, - B_CPU_PPC_IBM_401B2 = 36, - B_CPU_PPC_IBM_401C2 = 37, - B_CPU_PPC_IBM_401D2 = 38, - B_CPU_PPC_IBM_401E2 = 39, - B_CPU_PPC_IBM_401F2 = 40, - B_CPU_PPC_IBM_401G2 = 41, - B_CPU_PPC_IBM_403 = 42, - B_CPU_PPC_IBM_405GP = 43, - B_CPU_PPC_IBM_405L = 44, - B_CPU_PPC_IBM_750FX = 45, - B_CPU_PPC_IBM_POWER3 = 46, + uint32 cpu_count; /* number of cpus */ - /* Intel */ + uint64 max_pages; /* total # of accessible pages */ + uint64 used_pages; /* # of accessible pages in use */ + uint64 free_pages; + uint64 reserved_pages; + uint64 cached_pages; + uint64 block_cache_pages; + uint64 ignored_pages; /* # of ignored/inaccessible pages */ - /* Updated according to Intel(R) Processor Identification and - * the CPUID instruction (Table 4) - * AP-485 Intel - 24161832.pdf - */ - B_CPU_INTEL_x86 = 0x1000, - B_CPU_INTEL_PENTIUM = 0x1051, - B_CPU_INTEL_PENTIUM75, - B_CPU_INTEL_PENTIUM_486_OVERDRIVE, - B_CPU_INTEL_PENTIUM_MMX, - B_CPU_INTEL_PENTIUM_MMX_MODEL_4 = B_CPU_INTEL_PENTIUM_MMX, - B_CPU_INTEL_PENTIUM_MMX_MODEL_8 = 0x1058, - B_CPU_INTEL_PENTIUM75_486_OVERDRIVE, - B_CPU_INTEL_PENTIUM_PRO = 0x1061, - B_CPU_INTEL_PENTIUM_II = 0x1063, - B_CPU_INTEL_PENTIUM_II_MODEL_3 = 0x1063, - B_CPU_INTEL_PENTIUM_II_MODEL_5 = 0x1065, - B_CPU_INTEL_CELERON = 0x1066, - B_CPU_INTEL_CELERON_MODEL_22 = 0x11066, - B_CPU_INTEL_PENTIUM_III = 0x1067, - B_CPU_INTEL_PENTIUM_III_MODEL_8 = 0x1068, - B_CPU_INTEL_PENTIUM_M = 0x1069, - B_CPU_INTEL_PENTIUM_III_XEON = 0x106a, - B_CPU_INTEL_PENTIUM_III_MODEL_11 = 0x106b, - B_CPU_INTEL_ATOM = 0x1106c, - B_CPU_INTEL_PENTIUM_M_MODEL_13 = 0x106d, /* Dothan */ - B_CPU_INTEL_PENTIUM_CORE, - B_CPU_INTEL_PENTIUM_CORE_2, - B_CPU_INTEL_PENTIUM_CORE_2_45_NM = 0x11067, /* Core 2 on 45 nm - (Core 2 Extreme, - Xeon model 23 or - Core 2 Duo/Quad) */ - B_CPU_INTEL_PENTIUM_CORE_I5_M430 = 0x21065, /* Core i5 M 430 @ 2.27 */ - B_CPU_INTEL_PENTIUM_CORE_I7 = 0x1106a, /* Core i7 920 @ 2.6(6) */ - B_CPU_INTEL_PENTIUM_CORE_I7_Q720 = 0x1106e, /* Core i7 Q720 @ 1.6 */ - B_CPU_INTEL_PENTIUM_IV = 0x10f0, - B_CPU_INTEL_PENTIUM_IV_MODEL_1, - B_CPU_INTEL_PENTIUM_IV_MODEL_2, - B_CPU_INTEL_PENTIUM_IV_MODEL_3, - B_CPU_INTEL_PENTIUM_IV_MODEL_4, + uint64 max_swap_pages; + uint64 used_swap_pages; - /* AMD */ + uint32 page_faults; /* # of page faults */ - // AMD Processor Recognition Application Note - B_CPU_AMD_x86 = 0x1100, + uint32 max_sems; + uint32 used_sems; - // Family 5h - B_CPU_AMD_K5_MODEL_0 = 0x1150, - B_CPU_AMD_K5_MODEL_1 = 0x1151, - B_CPU_AMD_K5_MODEL_2 = 0x1152, - B_CPU_AMD_K5_MODEL_3 = 0x1153, - B_CPU_AMD_K6_MODEL_6 = 0x1156, - B_CPU_AMD_K6_MODEL_7 = 0x1157, - B_CPU_AMD_K6_MODEL_8 = 0x1158, - B_CPU_AMD_K6_2 = 0x1158, - B_CPU_AMD_K6_MODEL_9 = 0x1159, - B_CPU_AMD_K6_III = 0x1159, - B_CPU_AMD_K6_III_MODEL_13 = 0x115d, + uint32 max_ports; + uint32 used_ports; - B_CPU_AMD_GEODE_LX = 0x115a, + uint32 max_threads; + uint32 used_threads; - // Family 6h - B_CPU_AMD_ATHLON_MODEL_1 = 0x1161, - B_CPU_AMD_ATHLON_MODEL_2 = 0x1162, + uint32 max_teams; + uint32 used_teams; - B_CPU_AMD_DURON = 0x1163, + char kernel_name[B_FILE_NAME_LENGTH]; + char kernel_build_date[B_OS_NAME_LENGTH]; + char kernel_build_time[B_OS_NAME_LENGTH]; - B_CPU_AMD_ATHLON_THUNDERBIRD = 0x1164, - B_CPU_AMD_ATHLON_XP_MODEL_6 = 0x1166, - B_CPU_AMD_ATHLON_XP_MODEL_7 = 0x1167, - B_CPU_AMD_ATHLON_XP_MODEL_8 = 0x1168, - B_CPU_AMD_ATHLON_XP_MODEL_10 = 0x116a, /* Barton */ + int64 kernel_version; + uint32 abi; /* the system API */ +} system_info; - // Family fh - B_CPU_AMD_ATHLON_64_MODEL_3 = 0x11f3, - B_CPU_AMD_ATHLON_64_MODEL_4 = 0x11f4, - B_CPU_AMD_ATHLON_64_MODEL_7 = 0x11f7, - B_CPU_AMD_ATHLON_64_MODEL_8 = 0x11f8, - B_CPU_AMD_ATHLON_64_MODEL_11 = 0x11fb, - B_CPU_AMD_ATHLON_64_MODEL_12 = 0x11fc, - B_CPU_AMD_ATHLON_64_MODEL_14 = 0x11fe, - B_CPU_AMD_ATHLON_64_MODEL_15 = 0x11ff, - B_CPU_AMD_ATHLON_64_MODEL_20 = 0x111f4, - B_CPU_AMD_ATHLON_64_MODEL_23 = 0x111f7, - B_CPU_AMD_ATHLON_64_MODEL_24 = 0x111f8, - B_CPU_AMD_ATHLON_64_MODEL_27 = 0x111fb, - B_CPU_AMD_ATHLON_64_MODEL_28 = 0x111fc, - B_CPU_AMD_ATHLON_64_MODEL_31 = 0x111ff, - B_CPU_AMD_ATHLON_64_MODEL_35 = 0x211f3, - B_CPU_AMD_ATHLON_64_MODEL_43 = 0x211fb, - B_CPU_AMD_ATHLON_64_MODEL_44 = 0x211fc, - B_CPU_AMD_ATHLON_64_MODEL_47 = 0x211ff, - B_CPU_AMD_ATHLON_64_MODEL_63 = 0x311ff, - B_CPU_AMD_ATHLON_64_MODEL_79 = 0x411ff, - B_CPU_AMD_ATHLON_64_MODEL_95 = 0x511ff, - B_CPU_AMD_ATHLON_64_MODEL_127 = 0x711ff, +enum topology_level_type { + B_TOPOLOGY_UNKNOWN, + B_TOPOLOGY_ROOT, + B_TOPOLOGY_SMT, + B_TOPOLOGY_CORE, + B_TOPOLOGY_PACKAGE +}; - B_CPU_AMD_OPTERON_MODEL_5 = 0x11f5, - B_CPU_AMD_OPTERON_MODEL_21 = 0x111f5, - B_CPU_AMD_OPTERON_MODEL_33 = 0x211f1, - B_CPU_AMD_OPTERON_MODEL_37 = 0x211f5, - B_CPU_AMD_OPTERON_MODEL_39 = 0x211f7, - - B_CPU_AMD_TURION_64_MODEL_36 = 0x211f4, - B_CPU_AMD_TURION_64_MODEL_76 = 0x411fc, - B_CPU_AMD_TURION_64_MODEL_104 = 0x611f8, - - // Family 10h - B_CPU_AMD_PHENOM_MODEL_2 = 0x1011f2, - B_CPU_AMD_PHENOM_II_MODEL_4 = 0x1011f4, - B_CPU_AMD_PHENOM_II_MODEL_5 = 0x1011f5, - B_CPU_AMD_PHENOM_II_MODEL_6 = 0x1011f6, - B_CPU_AMD_PHENOM_II_MODEL_10 = 0x1011fa, - - // Family 12h - B_CPU_AMD_A_SERIES = 0x3011f1, - - // Family 14h - B_CPU_AMD_C_SERIES = 0x5011f1, - B_CPU_AMD_E_SERIES = 0x5011f2, - - // Family 15h - B_CPU_AMD_FX_SERIES_MODEL_1 = 0x6011f1, /* Bulldozer */ - B_CPU_AMD_FX_SERIES_MODEL_2 = 0x6011f2, - - /* VIA/Cyrix */ - B_CPU_CYRIX_x86 = 0x1200, - B_CPU_VIA_CYRIX_x86 = 0x1200, - B_CPU_CYRIX_GXm = 0x1254, - B_CPU_CYRIX_6x86MX = 0x1260, - - /* VIA/IDT */ - B_CPU_IDT_x86 = 0x1300, - B_CPU_VIA_IDT_x86 = 0x1300, - B_CPU_IDT_WINCHIP_C6 = 0x1354, - B_CPU_IDT_WINCHIP_2 = 0x1358, - B_CPU_IDT_WINCHIP_3, - B_CPU_VIA_C3_SAMUEL = 0x1366, - B_CPU_VIA_C3_SAMUEL_2 = 0x1367, - B_CPU_VIA_C3_EZRA_T = 0x1368, - B_CPU_VIA_C3_NEHEMIAH = 0x1369, - B_CPU_VIA_C7_ESTHER = 0x136a, - B_CPU_VIA_C7_ESTHER_2 = 0x136d, - B_CPU_VIA_NANO_ISAIAH = 0x136f, - - /* Transmeta */ - B_CPU_TRANSMETA_x86 = 0x1600, - B_CPU_TRANSMETA_CRUSOE = 0x1654, - B_CPU_TRANSMETA_EFFICEON = 0x16f2, - B_CPU_TRANSMETA_EFFICEON_2 = 0x16f3, - - /* Rise */ - B_CPU_RISE_x86 = 0x1400, - B_CPU_RISE_mP6 = 0x1450, - - /* National Semiconductor */ - B_CPU_NATIONAL_x86 = 0x1500, - B_CPU_NATIONAL_GEODE_GX1 = 0x1554, - B_CPU_NATIONAL_GEODE_GX2, - - /* For compatibility */ - B_CPU_AMD_29K = 14, +enum cpu_platform { + B_CPU_UNKNOWN, B_CPU_x86, - B_CPU_MC6502, - B_CPU_Z80, - B_CPU_ALPHA, - B_CPU_MIPS, - B_CPU_HPPA, - B_CPU_M68K, - B_CPU_ARM, - B_CPU_SH, - B_CPU_SPARC -} cpu_type; + B_CPU_x86_64 +}; + +enum cpu_vendor { + B_CPU_VENDOR_UNKNOWN, + B_CPU_VENDOR_AMD, + B_CPU_VENDOR_CYRIX, + B_CPU_VENDOR_IDT, + B_CPU_VENDOR_INTEL, + B_CPU_VENDOR_NATIONAL_SEMICONDUCTOR, + B_CPU_VENDOR_RISE, + B_CPU_VENDOR_TRANSMETA, + B_CPU_VENDOR_VIA +}; + +typedef struct { + enum cpu_platform platform; +} cpu_topology_root_info; + +typedef struct { + enum cpu_vendor vendor; + uint32 cache_line_size; +} cpu_topology_package_info; + +typedef struct { + uint32 model; + uint64 default_frequency; +} cpu_topology_core_info; + +typedef struct { + uint32 id; + enum topology_level_type type; + uint32 level; + + union { + cpu_topology_root_info root; + cpu_topology_package_info package; + cpu_topology_core_info core; + } data; +} cpu_topology_node_info; + + +extern status_t get_system_info(system_info* info); +extern status_t get_cpu_info(uint32 firstCPU, uint32 cpuCount, + cpu_info* info); +extern status_t get_cpu_topology_info(cpu_topology_node_info* topologyInfos, + uint32* topologyInfoCount); #define B_CPU_x86_VENDOR_MASK 0xff00 @@ -689,76 +577,6 @@ extern status_t get_cpuid(cpuid_info *info, uint32 eaxRegister, #endif -typedef enum platform_types { - B_BEBOX_PLATFORM = 0, - B_MAC_PLATFORM, - B_AT_CLONE_PLATFORM, - B_ENIAC_PLATFORM, - B_APPLE_II_PLATFORM, - B_CRAY_PLATFORM, - B_LISA_PLATFORM, - B_TI_994A_PLATFORM, - B_TIMEX_SINCLAIR_PLATFORM, - B_ORAC_1_PLATFORM, - B_HAL_PLATFORM, - B_BESM_6_PLATFORM, - B_MK_61_PLATFORM, - B_NINTENDO_64_PLATFORM, - B_AMIGA_PLATFORM, - B_ATARI_PLATFORM, - B_64_BIT_PC_PLATFORM -} platform_type; - -typedef struct { - bigtime_t active_time; /* usec of doing useful work since boot */ -} cpu_info; - - -typedef int32 machine_id[2]; /* unique machine ID */ - -typedef struct { - machine_id id; /* unique machine ID */ - bigtime_t boot_time; /* time of boot (usecs since 1/1/1970) */ - - int32 cpu_count; /* number of cpus */ - enum cpu_types cpu_type; /* type of cpu */ - int32 cpu_revision; /* revision # of cpu */ - cpu_info cpu_infos[8]; /* info about individual cpus */ - int64 cpu_clock_speed; /* processor clock speed (Hz) */ - int64 bus_clock_speed; /* bus clock speed (Hz) */ - enum platform_types platform_type; /* type of machine we're on */ - - int32 max_pages; /* total # of accessible pages */ - int32 used_pages; /* # of accessible pages in use */ - int32 page_faults; /* # of page faults */ - int32 max_sems; - int32 used_sems; - int32 max_ports; - int32 used_ports; - int32 max_threads; - int32 used_threads; - int32 max_teams; - int32 used_teams; - - char kernel_name[B_FILE_NAME_LENGTH]; - char kernel_build_date[B_OS_NAME_LENGTH]; - char kernel_build_time[B_OS_NAME_LENGTH]; - int64 kernel_version; - - bigtime_t _busy_wait_time; /* reserved for whatever */ - - int32 cached_pages; - uint32 abi; /* the system API */ - int32 ignored_pages; /* # of ignored/inaccessible pages */ - int32 pad; -} system_info; - -/* system private, use macro instead */ -extern status_t _get_system_info(system_info *info, size_t size); - -#define get_system_info(info) \ - _get_system_info((info), sizeof(*(info))) - extern int32 is_computer_on(void); extern double is_computer_on_fire(void); diff --git a/headers/private/kernel/arch/system_info.h b/headers/private/kernel/arch/system_info.h index 3a5fb24c8e..f765be64bd 100644 --- a/headers/private/kernel/arch/system_info.h +++ b/headers/private/kernel/arch/system_info.h @@ -18,7 +18,7 @@ extern "C" { #endif status_t arch_system_info_init(struct kernel_args *args); -status_t arch_get_system_info(system_info *info, size_t size); +void arch_fill_topology_node(cpu_topology_node_info* node, int32 cpu); #ifdef __cplusplus } diff --git a/headers/private/kernel/ksystem_info.h b/headers/private/kernel/ksystem_info.h index 9ff900cd33..1fc466d8ee 100644 --- a/headers/private/kernel/ksystem_info.h +++ b/headers/private/kernel/ksystem_info.h @@ -21,7 +21,11 @@ status_t system_info_init(struct kernel_args *args); status_t system_notifications_init(); const char* get_haiku_revision(void); -status_t _user_get_system_info(system_info *userInfo, size_t size); +status_t _user_get_system_info(system_info *userInfo); +status_t _user_get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info); +status_t _user_get_cpu_topology_info(cpu_topology_node_info* topologyInfos, + uint32* topologyInfoCount); + status_t _user_get_system_info_etc(int32 id, void *buffer, size_t bufferSize); diff --git a/headers/private/shared/cpu_type.h b/headers/private/shared/cpu_type.h index d71983e14f..b84fa694bd 100644 --- a/headers/private/shared/cpu_type.h +++ b/headers/private/shared/cpu_type.h @@ -1,5 +1,6 @@ /* * Copyright 2004-2013, Axel Dörfler, axeld@pinc-software.de. + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Distributed under the terms of the MIT License. */ @@ -18,8 +19,9 @@ extern "C" { #endif -const char *get_cpu_vendor_string(enum cpu_types type); -const char *get_cpu_model_string(const system_info *info); +static const char* get_cpu_vendor_string(enum cpu_vendor cpuVendor); +static const char* get_cpu_model_string(enum cpu_platform platform, + enum cpu_vendor cpuVendor, uint32 cpuModel); void get_cpu_type(char *vendorBuffer, size_t vendorSize, char *modelBuffer, size_t modelSize); int32 get_rounded_cpu_speed(void); @@ -165,36 +167,16 @@ parse_amd(const char* name) #endif -const char * -get_cpu_vendor_string(enum cpu_types type) +static const char* +get_cpu_vendor_string(enum cpu_vendor cpuVendor) { -#if __POWERPC__ - /* We're not that nice here. */ - return "IBM/Motorola"; -#endif -#if defined(__INTEL__) || defined(__x86_64__) - /* Determine x86 vendor name */ - switch (type & B_CPU_x86_VENDOR_MASK) { - case B_CPU_INTEL_x86: - return "Intel"; - case B_CPU_AMD_x86: - return "AMD"; - case B_CPU_CYRIX_x86: - return "Cyrix"; - case B_CPU_IDT_x86: - /* IDT was bought by VIA */ - if (((type >> 8) & 0xf) >= 6) - return "VIA"; - return "IDT"; - case B_CPU_RISE_x86: - return "Rise"; - case B_CPU_TRANSMETA_x86: - return "Transmeta"; + static const char* vendorStrings[] = { + NULL, "AMD", "Cyrix", "IDT", "Intel", "Rise", "Transmeta", "VIA", + }; - default: - return NULL; - } -#endif + if ((size_t)cpuVendor >= sizeof(vendorStrings) / sizeof(const char*)) + return NULL; + return vendorStrings[cpuVendor]; } @@ -256,248 +238,217 @@ get_cpuid_model_string(char *name) #endif /* __INTEL__ || __x86_64__ */ -const char * -get_cpu_model_string(const system_info *info) +static const char* +get_cpu_model_string(enum cpu_platform platform, enum cpu_vendor cpuVendor, + uint32 cpuModel) { #if defined(__INTEL__) || defined(__x86_64__) char cpuidName[49]; - /* for use with get_cpuid_model_string() */ -#endif /* __INTEL__ || __x86_64__ */ +#endif + + (void)cpuVendor; + (void)cpuModel; - /* Determine CPU type */ - switch (info->cpu_type) { -#if __POWERPC__ - case B_CPU_PPC_603: - return "603"; - case B_CPU_PPC_603e: - return "603e"; - case B_CPU_PPC_750: - return "750"; - case B_CPU_PPC_604: - return "604"; - case B_CPU_PPC_604e: - return "604e"; - default: - return NULL; -#endif /* __POWERPC__ */ #if defined(__INTEL__) || defined(__x86_64__) - case B_CPU_x86: - return "Unknown x86"; + if (platform != B_CPU_x86 && platform != B_CPU_x86_64) + return NULL; - /* Intel */ - case B_CPU_INTEL_PENTIUM: - case B_CPU_INTEL_PENTIUM75: - return "Pentium"; - case B_CPU_INTEL_PENTIUM_486_OVERDRIVE: - case B_CPU_INTEL_PENTIUM75_486_OVERDRIVE: - return "Pentium OD"; - case B_CPU_INTEL_PENTIUM_MMX: - case B_CPU_INTEL_PENTIUM_MMX_MODEL_8: - return "Pentium MMX"; - case B_CPU_INTEL_PENTIUM_PRO: - return "Pentium Pro"; - case B_CPU_INTEL_PENTIUM_II_MODEL_3: - case B_CPU_INTEL_PENTIUM_II_MODEL_5: - return "Pentium II"; - case B_CPU_INTEL_CELERON: - case B_CPU_INTEL_CELERON_MODEL_22: - return "Celeron"; - case B_CPU_INTEL_PENTIUM_III: - case B_CPU_INTEL_PENTIUM_III_MODEL_8: - case B_CPU_INTEL_PENTIUM_III_MODEL_11: - case B_CPU_INTEL_PENTIUM_III_XEON: - return "Pentium III"; - case B_CPU_INTEL_PENTIUM_M: - case B_CPU_INTEL_PENTIUM_M_MODEL_13: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Celeron") != NULL) - return "Pentium M Celeron"; - return "Pentium M"; - case B_CPU_INTEL_ATOM: - return "Atom"; - case B_CPU_INTEL_PENTIUM_CORE: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Celeron") != NULL) - return "Core Celeron"; - return "Core"; - case B_CPU_INTEL_PENTIUM_CORE_2: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Celeron") != NULL) - return "Core 2 Celeron"; - if (strcasestr(cpuidName, "Xeon") != NULL) - return "Core 2 Xeon"; - return "Core 2"; - case B_CPU_INTEL_PENTIUM_CORE_2_45_NM: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Celeron") != NULL) - return "Core 2 Celeron"; - if (strcasestr(cpuidName, "Xeon") != NULL) - return "Core 2 Xeon"; - if (strcasestr(cpuidName, "Pentium") != NULL) - return "Pentium"; - if (strcasestr(cpuidName, "Extreme") != NULL) - return "Core 2 Extreme"; - return "Core 2"; - case B_CPU_INTEL_PENTIUM_CORE_I5_M430: - return "Core i5"; - case B_CPU_INTEL_PENTIUM_CORE_I7: - case B_CPU_INTEL_PENTIUM_CORE_I7_Q720: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Xeon") != NULL) - return "Core i7 Xeon"; - return "Core i7"; - case B_CPU_INTEL_PENTIUM_IV: - case B_CPU_INTEL_PENTIUM_IV_MODEL_1: - case B_CPU_INTEL_PENTIUM_IV_MODEL_2: - case B_CPU_INTEL_PENTIUM_IV_MODEL_3: - case B_CPU_INTEL_PENTIUM_IV_MODEL_4: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Celeron") != NULL) - return "Pentium 4 Celeron"; - if (strcasestr(cpuidName, "Xeon") != NULL) - return "Pentium 4 Xeon"; - return "Pentium 4"; + uint16 family = ((cpuModel >> 8) & 0xf) | ((cpuModel >> 16) & 0xff0); + uint16 model = ((cpuModel >> 4) & 0xf) | ((cpuModel >> 12) & 0xf0); + uint8 stepping = cpuModel & 0xf; - /* AMD */ - case B_CPU_AMD_K5_MODEL_0: - case B_CPU_AMD_K5_MODEL_1: - case B_CPU_AMD_K5_MODEL_2: - case B_CPU_AMD_K5_MODEL_3: - return "K5"; - case B_CPU_AMD_K6_MODEL_6: - case B_CPU_AMD_K6_MODEL_7: - return "K6"; - case B_CPU_AMD_K6_2: - return "K6-2"; - case B_CPU_AMD_K6_III: - case B_CPU_AMD_K6_III_MODEL_13: - return "K6-III"; - case B_CPU_AMD_GEODE_LX: - return "Geode LX"; - case B_CPU_AMD_ATHLON_MODEL_1: - case B_CPU_AMD_ATHLON_MODEL_2: - case B_CPU_AMD_ATHLON_THUNDERBIRD: - return "Athlon"; - case B_CPU_AMD_ATHLON_XP_MODEL_6: - case B_CPU_AMD_ATHLON_XP_MODEL_7: - case B_CPU_AMD_ATHLON_XP_MODEL_8: - case B_CPU_AMD_ATHLON_XP_MODEL_10: - return "Athlon XP"; - case B_CPU_AMD_DURON: - return "Duron"; - case B_CPU_AMD_ATHLON_64_MODEL_3: - case B_CPU_AMD_ATHLON_64_MODEL_4: - case B_CPU_AMD_ATHLON_64_MODEL_7: - case B_CPU_AMD_ATHLON_64_MODEL_8: - case B_CPU_AMD_ATHLON_64_MODEL_11: - case B_CPU_AMD_ATHLON_64_MODEL_12: - case B_CPU_AMD_ATHLON_64_MODEL_14: - case B_CPU_AMD_ATHLON_64_MODEL_15: - case B_CPU_AMD_ATHLON_64_MODEL_20: - case B_CPU_AMD_ATHLON_64_MODEL_23: - case B_CPU_AMD_ATHLON_64_MODEL_24: - case B_CPU_AMD_ATHLON_64_MODEL_27: - case B_CPU_AMD_ATHLON_64_MODEL_28: - case B_CPU_AMD_ATHLON_64_MODEL_31: - case B_CPU_AMD_ATHLON_64_MODEL_35: - case B_CPU_AMD_ATHLON_64_MODEL_43: - case B_CPU_AMD_ATHLON_64_MODEL_44: - case B_CPU_AMD_ATHLON_64_MODEL_47: - case B_CPU_AMD_ATHLON_64_MODEL_63: - case B_CPU_AMD_ATHLON_64_MODEL_79: - case B_CPU_AMD_ATHLON_64_MODEL_95: - case B_CPU_AMD_ATHLON_64_MODEL_127: - return "Athlon 64"; - case B_CPU_AMD_OPTERON_MODEL_5: - case B_CPU_AMD_OPTERON_MODEL_21: - case B_CPU_AMD_OPTERON_MODEL_33: - case B_CPU_AMD_OPTERON_MODEL_37: - case B_CPU_AMD_OPTERON_MODEL_39: - return "Opteron"; - case B_CPU_AMD_TURION_64_MODEL_36: - case B_CPU_AMD_TURION_64_MODEL_76: - case B_CPU_AMD_TURION_64_MODEL_104: - return "Turion 64"; - case B_CPU_AMD_PHENOM_MODEL_2: - return "Phenom"; - case B_CPU_AMD_PHENOM_II_MODEL_4: - case B_CPU_AMD_PHENOM_II_MODEL_5: - case B_CPU_AMD_PHENOM_II_MODEL_6: - case B_CPU_AMD_PHENOM_II_MODEL_10: - get_cpuid_model_string(cpuidName); - if (strcasestr(cpuidName, "Athlon") != NULL) - return "Athlon II"; - return "Phenom II"; - case B_CPU_AMD_A_SERIES: + if (cpuVendor == B_CPU_VENDOR_AMD) { + if (family == 5) { + if (model <= 3) + return "K5"; + if (model <= 7) + return "K6"; + if (model == 8) + return "K6-2"; + if (model == 9 || model == 0xd) + return "K6-III"; + if (model == 0xa) + return "Geode LX"; + } else if (family == 6) { + if (model <= 2 || model == 4) + return "Athlon"; + if (model == 3) + return "Duron"; + if (model <= 8 || model == 0xa) + return "Athlon XP"; + } else if (family == 0xf) { + if (model <= 4 || model == 7 || model == 8 + || (model >= 0xb && model <= 0xf) || model == 0x14 + || model == 0x18 || model == 0x1b || model == 0x1c + || model == 0x1f || model == 0x23 || model == 0x2b + || model == 0x2c + || ((model & 0xf) == 0xf && model >= 0x2f && model <= 0x7f)) { + return "Athlon 64"; + } + if (model == 5 || model == 0x15 || model == 0x21 || model == 0x25 + || model == 0x27) { + return "Opteron"; + } + if (model == 0x24 || model == 0x4c || model == 0x68) + return "Turion 64"; + } else if (family == 0x1f) { + if (model == 2) + return "Phenom"; + if ((model >= 4 && model <= 6) || model == 0xa) { + get_cpuid_model_string(cpuidName); + if (strcasestr(cpuidName, "Athlon") != NULL) + return "Athlon II"; + return "Phenom II"; + } + } else if (family == 0x3f) return "A-Series"; - case B_CPU_AMD_C_SERIES: - return "C-Series"; - case B_CPU_AMD_E_SERIES: - return "E-Series"; - case B_CPU_AMD_FX_SERIES_MODEL_1: - case B_CPU_AMD_FX_SERIES_MODEL_2: + else if (family == 0x5f) { + if (model == 1) + return "C-Series"; + if (model == 2) + return "E-Series"; + } else if (family == 0x6f) return "FX-Series"; - /* Transmeta */ - case B_CPU_TRANSMETA_CRUSOE: - return "Crusoe"; - case B_CPU_TRANSMETA_EFFICEON: - case B_CPU_TRANSMETA_EFFICEON_2: - return "Efficeon"; - - /* IDT/VIA */ - case B_CPU_IDT_WINCHIP_C6: - return "WinChip C6"; - case B_CPU_IDT_WINCHIP_2: - return "WinChip 2"; - case B_CPU_VIA_C3_SAMUEL: - return "C3 Samuel"; - case B_CPU_VIA_C3_SAMUEL_2: - /* stepping identified the model */ - if ((info->cpu_revision & 0xf) < 8) - return "C3 Eden/Samuel 2"; - return "C3 Ezra"; - case B_CPU_VIA_C3_EZRA_T: - return "C3 Ezra-T"; - case B_CPU_VIA_C3_NEHEMIAH: - /* stepping identified the model */ - if ((info->cpu_revision & 0xf) < 8) - return "C3 Nehemiah"; - return "C3 Eden-N"; - case B_CPU_VIA_C7_ESTHER: - case B_CPU_VIA_C7_ESTHER_2: - return "C7"; - case B_CPU_VIA_NANO_ISAIAH: - return "Nano"; - - /* Cyrix/VIA */ - case B_CPU_CYRIX_GXm: - return "GXm"; - case B_CPU_CYRIX_6x86MX: - return "6x86MX"; - - /* Rise */ - case B_CPU_RISE_mP6: - return "mP6"; - - /* National Semiconductor */ - case B_CPU_NATIONAL_GEODE_GX1: - return "Geode GX1"; - - default: - if ((info->cpu_type & B_CPU_x86_VENDOR_MASK) == B_CPU_INTEL_x86) { - // Fallback to manual parsing of the model string - get_cpuid_model_string(cpuidName); - return parse_intel(cpuidName); - } - if ((info->cpu_type & B_CPU_x86_VENDOR_MASK) == B_CPU_AMD_x86) { - // Fallback to manual parsing of the model string - get_cpuid_model_string(cpuidName); - return parse_amd(cpuidName); - } - return NULL; -#endif /* __INTEL__ || __x86_64__ */ + // Fallback to manual parsing of the model string + get_cpuid_model_string(cpuidName); + return parse_amd(cpuidName); } + + if (cpuVendor == B_CPU_VENDOR_CYRIX) { + if (family == 5 && model == 4) + return "GXm"; + if (family == 6) + return "6x86MX"; + return NULL; + } + + if (cpuVendor == B_CPU_VENDOR_INTEL) { + if (family == 5) { + if (model == 1 || model == 2) + return "Pentium"; + if (model == 3 || model == 9) + return "Pentium OD"; + if (model == 4 || model == 8) + return "Pentium MMX"; + } else if (family == 6) { + if (model == 1) + return "Pentium Pro"; + if (model == 3 || model == 5) + return "Pentium II"; + if (model == 6) + return "Celeron"; + if (model == 7 || model == 8 || model == 0xa || model == 0xb) + return "Pentium III"; + if (model == 9 || model == 0xd) { + get_cpuid_model_string(cpuidName); + if (strcasestr(cpuidName, "Celeron") != NULL) + return "Pentium M Celeron"; + return "Pentium M"; + } + if (model == 0x1c || model == 0x26 || model == 0x36) + return "Atom"; + if (model == 0xe) { + get_cpuid_model_string(cpuidName); + if (strcasestr(cpuidName, "Celeron") != NULL) + return "Core Celeron"; + return "Core"; + } + if (model == 0xf || model == 0x17) { + get_cpuid_model_string(cpuidName); + if (strcasestr(cpuidName, "Celeron") != NULL) + return "Core 2 Celeron"; + if (strcasestr(cpuidName, "Xeon") != NULL) + return "Core 2 Xeon"; + if (strcasestr(cpuidName, "Pentium") != NULL) + return "Pentium"; + if (strcasestr(cpuidName, "Extreme") != NULL) + return "Core 2 Extreme"; + return "Core 2"; + } + if (model == 0x25) + return "Core i5"; + if (model == 0x1a || model == 0x1e) { + get_cpuid_model_string(cpuidName); + if (strcasestr(cpuidName, "Xeon") != NULL) + return "Core i7 Xeon"; + return "Core i7"; + } + } else if (family == 0xf) { + if (model <= 4) { + get_cpuid_model_string(cpuidName); + if (strcasestr(cpuidName, "Celeron") != NULL) + return "Pentium 4 Celeron"; + if (strcasestr(cpuidName, "Xeon") != NULL) + return "Pentium 4 Xeon"; + return "Pentium 4"; + } + } + + // Fallback to manual parsing of the model string + get_cpuid_model_string(cpuidName); + return parse_intel(cpuidName); + } + + if (cpuVendor == B_CPU_VENDOR_NATIONAL_SEMICONDUCTOR) { + if (family == 5) { + if (model == 4) + return "Geode GX1"; + if (model == 5) + return "Geode GX2"; + return NULL; + } + } + + if (cpuVendor == B_CPU_VENDOR_RISE) { + if (family == 5) + return "mP6"; + return NULL; + } + + if (cpuVendor == B_CPU_VENDOR_TRANSMETA) { + if (family == 5 && model == 4) + return "Crusoe"; + if (family == 0xf && (model == 2 || model == 3)) + return "Efficeon"; + return NULL; + } + + if (cpuVendor == B_CPU_VENDOR_VIA) { + if (family == 5) { + if (model == 4) + return "WinChip C6"; + if (model == 8) + return "WinChip 2"; + if (model == 9) + return "WinChip 3"; + return NULL; + } else if (family == 6) { + if (model == 6) + return "C3 Samuel"; + if (model == 7) { + if (stepping < 8) + return "C3 Eden/Samuel 2"; + return "C3 Ezra"; + } + if (model == 8) + return "C3 Ezra-T"; + if (model == 9) { + if (stepping < 8) + return "C3 Nehemiah"; + return "C3 Ezra-N"; + } + if (model == 0xa || model == 0xd) + return "C7"; + if (model == 0xf) + return "Nano"; + return NULL; + } + } + +#endif + + return NULL; } @@ -506,15 +457,42 @@ get_cpu_type(char *vendorBuffer, size_t vendorSize, char *modelBuffer, size_t modelSize) { const char *vendor, *model; - system_info info; - get_system_info(&info); + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + get_cpu_topology_info(NULL, &topologyNodeCount); + if (topologyNodeCount != 0) + topology = new cpu_topology_node_info[topologyNodeCount]; + get_cpu_topology_info(topology, &topologyNodeCount); - vendor = get_cpu_vendor_string(info.cpu_type); + enum cpu_platform platform = B_CPU_UNKNOWN; + enum cpu_vendor cpuVendor = B_CPU_VENDOR_UNKNOWN; + uint32 cpuModel = 0; + for (uint32 i = 0; i < topologyNodeCount; i++) { + switch (topology[i].type) { + case B_TOPOLOGY_ROOT: + platform = topology[i].data.root.platform; + break; + + case B_TOPOLOGY_PACKAGE: + cpuVendor = topology[i].data.package.vendor; + break; + + case B_TOPOLOGY_CORE: + cpuModel = topology[i].data.core.model; + break; + + default: + break; + } + } + delete[] topology; + + vendor = get_cpu_vendor_string(cpuVendor); if (vendor == NULL) vendor = "Unknown"; - model = get_cpu_model_string(&info); + model = get_cpu_model_string(platform, cpuVendor, cpuModel); if (model == NULL) model = "Unknown"; @@ -533,14 +511,27 @@ get_cpu_type(char *vendorBuffer, size_t vendorSize, char *modelBuffer, int32 get_rounded_cpu_speed(void) { - system_info info; + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + get_cpu_topology_info(NULL, &topologyNodeCount); + if (topologyNodeCount != 0) + topology = new cpu_topology_node_info[topologyNodeCount]; + get_cpu_topology_info(topology, &topologyNodeCount); + + uint64 cpuFrequency = 0; + for (uint32 i = 0; i < topologyNodeCount; i++) { + if (topology[i].type == B_TOPOLOGY_CORE) { + cpuFrequency = topology[i].data.core.default_frequency; + break; + } + } + delete[] topology; int target, frac, delta; int freqs[] = { 100, 50, 25, 75, 33, 67, 20, 40, 60, 80, 10, 30, 70, 90 }; uint x; - get_system_info(&info); - target = info.cpu_clock_speed / 1000000; + target = cpuFrequency / 1000000; frac = target % 100; delta = -frac; diff --git a/headers/private/system/syscalls.h b/headers/private/system/syscalls.h index 6b1abb6b0b..fd787c4cda 100644 --- a/headers/private/system/syscalls.h +++ b/headers/private/system/syscalls.h @@ -514,7 +514,13 @@ extern int64 _kern_atomic_get64(vint64 *value); #endif // ATOMIC64_FUNCS_ARE_SYSCALLS /* System informations */ -extern status_t _kern_get_system_info(system_info *info, size_t size); +extern status_t _kern_get_system_info(system_info* info); +extern status_t _kern_get_cpu_info(uint32 firstCPU, uint32 cpuCount, + cpu_info* info); +extern status_t _kern_get_cpu_topology_info( + cpu_topology_node_info* topologyInfos, + uint32* topologyInfoCount); + extern status_t _kern_get_system_info_etc(int32 id, void *buffer, size_t bufferSize); extern status_t _kern_analyze_scheduling(bigtime_t from, bigtime_t until, diff --git a/headers/private/system/system_info.h b/headers/private/system/system_info.h index 888c4423c8..89a47f6bac 100644 --- a/headers/private/system/system_info.h +++ b/headers/private/system/system_info.h @@ -63,6 +63,10 @@ extern "C" { #endif +status_t __get_system_info(system_info* info); +status_t __get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info); +status_t __get_cpu_topology_info(cpu_topology_node_info* topologyInfos, + uint32* topologyInfoCount); status_t __get_system_info_etc(int32 id, void* buffer, size_t bufferSize); status_t __start_watching_system(int32 object, uint32 flags, port_id port, diff --git a/src/apps/aboutsystem/AboutSystem.cpp b/src/apps/aboutsystem/AboutSystem.cpp index 48e71973fd..1748fabc95 100644 --- a/src/apps/aboutsystem/AboutSystem.cpp +++ b/src/apps/aboutsystem/AboutSystem.cpp @@ -615,9 +615,40 @@ AboutView::AboutView() strlcpy(processorLabel, B_TRANSLATE("Processor:"), sizeof(processorLabel)); + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + get_cpu_topology_info(NULL, &topologyNodeCount); + if (topologyNodeCount != 0) + topology = new cpu_topology_node_info[topologyNodeCount]; + get_cpu_topology_info(topology, &topologyNodeCount); + + enum cpu_platform platform = B_CPU_UNKNOWN; + enum cpu_vendor cpuVendor = B_CPU_VENDOR_UNKNOWN; + uint32 cpuModel = 0; + for (uint32 i = 0; i < topologyNodeCount; i++) { + switch (topology[i].type) { + case B_TOPOLOGY_ROOT: + platform = topology[i].data.root.platform; + break; + + case B_TOPOLOGY_PACKAGE: + cpuVendor = topology[i].data.package.vendor; + break; + + case B_TOPOLOGY_CORE: + cpuModel = topology[i].data.core.model; + break; + + default: + break; + } + } + + delete[] topology; + BString cpuType; - cpuType << get_cpu_vendor_string(systemInfo.cpu_type) - << " " << get_cpu_model_string(&systemInfo); + cpuType << get_cpu_vendor_string(cpuVendor) + << " " << get_cpu_model_string(platform, cpuVendor, cpuModel); BStringView* cpuView = new BStringView("cputext", cpuType.String()); cpuView->SetExplicitAlignment(BAlignment(B_ALIGN_LEFT, diff --git a/src/apps/activitymonitor/DataSource.cpp b/src/apps/activitymonitor/DataSource.cpp index b7b15f0d24..9c954e1051 100644 --- a/src/apps/activitymonitor/DataSource.cpp +++ b/src/apps/activitymonitor/DataSource.cpp @@ -873,7 +873,7 @@ CPUUsageDataSource::Print(BString& text, int64 value) const int64 CPUUsageDataSource::NextValue(SystemInfo& info) { - bigtime_t active = info.Info().cpu_infos[fCPU].active_time; + bigtime_t active = info.CPUActiveTime(fCPU); int64 percent = int64(1000.0 * (active - fPreviousActive) / (info.Time() - fPreviousTime)); @@ -1018,8 +1018,8 @@ CPUCombinedUsageDataSource::NextValue(SystemInfo& info) int32 running = 0; bigtime_t active = 0; - for (int32 cpu = 0; cpu < info.Info().cpu_count; cpu++) { - active += info.Info().cpu_infos[cpu].active_time; + for (uint32 cpu = 0; cpu < info.CPUCount(); cpu++) { + active += info.CPUActiveTime(cpu); running++; // TODO: take disabled CPUs into account } diff --git a/src/apps/activitymonitor/SystemInfo.cpp b/src/apps/activitymonitor/SystemInfo.cpp index 423c2bd72d..2416e4a2a0 100644 --- a/src/apps/activitymonitor/SystemInfo.cpp +++ b/src/apps/activitymonitor/SystemInfo.cpp @@ -24,6 +24,8 @@ SystemInfo::SystemInfo(SystemInfoHandler* handler) fMediaBuffers(0) { get_system_info(&fSystemInfo); + fCPUInfos = new cpu_info[fSystemInfo.cpu_count]; + get_cpu_info(0, fSystemInfo.cpu_count, fCPUInfos); __get_system_info_etc(B_MEMORY_INFO, &fMemoryInfo, sizeof(system_memory_info)); @@ -40,6 +42,7 @@ SystemInfo::SystemInfo(SystemInfoHandler* handler) SystemInfo::~SystemInfo() { + delete[] fCPUInfos; } diff --git a/src/apps/activitymonitor/SystemInfo.h b/src/apps/activitymonitor/SystemInfo.h index 2bee16c708..66e900b27b 100644 --- a/src/apps/activitymonitor/SystemInfo.h +++ b/src/apps/activitymonitor/SystemInfo.h @@ -42,6 +42,8 @@ public: bigtime_t Time() const { return fTime; } uint32 CPUCount() const { return fSystemInfo.cpu_count; } + bigtime_t CPUActiveTime(uint32 cpu) const + { return fCPUInfos[cpu].active_time; } const system_info& Info() const { return fSystemInfo; } uint64 NetworkReceived(); @@ -61,6 +63,7 @@ private: void _RetrieveNetwork(); system_info fSystemInfo; + cpu_info* fCPUInfos; system_memory_info fMemoryInfo; bigtime_t fTime; bool fRetrievedNetwork; diff --git a/src/apps/debugger/controllers/DebugReportGenerator.cpp b/src/apps/debugger/controllers/DebugReportGenerator.cpp index bf45375753..476b99e4ff 100644 --- a/src/apps/debugger/controllers/DebugReportGenerator.cpp +++ b/src/apps/debugger/controllers/DebugReportGenerator.cpp @@ -234,11 +234,40 @@ DebugReportGenerator::_GenerateReportHeader(BString& _output) SystemInfo sysInfo; + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + get_cpu_topology_info(NULL, &topologyNodeCount); + if (topologyNodeCount != 0) + topology = new cpu_topology_node_info[topologyNodeCount]; + get_cpu_topology_info(topology, &topologyNodeCount); + + cpu_platform platform = B_CPU_UNKNOWN; + cpu_vendor cpuVendor = B_CPU_VENDOR_UNKNOWN; + uint32 cpuModel = 0; + for (uint32 i = 0; i < topologyNodeCount; i++) { + switch (topology[i].type) { + case B_TOPOLOGY_ROOT: + platform = topology[i].data.root.platform; + break; + + case B_TOPOLOGY_PACKAGE: + cpuVendor = topology[i].data.package.vendor; + break; + + case B_TOPOLOGY_CORE: + cpuModel = topology[i].data.core.model; + break; + + default: + break; + } + } + if (fDebuggerInterface->GetSystemInfo(sysInfo) == B_OK) { const system_info &info = sysInfo.GetSystemInfo(); data.SetToFormat("CPU(s): %" B_PRId32 "x %s %s\n", - info.cpu_count, get_cpu_vendor_string(info.cpu_type), - get_cpu_model_string(&info)); + info.cpu_count, get_cpu_vendor_string(cpuVendor), + get_cpu_model_string(platform, cpuVendor, cpuModel)); _output << data; char maxSize[32]; char usedSize[32]; @@ -256,6 +285,7 @@ DebugReportGenerator::_GenerateReportHeader(BString& _output) _output << data; } + delete[] topology; return B_OK; } diff --git a/src/apps/diskprobe/TypeEditors.cpp b/src/apps/diskprobe/TypeEditors.cpp index 39bfbf8cb2..3cf2d46948 100644 --- a/src/apps/diskprobe/TypeEditors.cpp +++ b/src/apps/diskprobe/TypeEditors.cpp @@ -187,7 +187,7 @@ TypeEditorView::TypeMatches() system_info info; get_system_info(&info); - return fEditor.FileSize() / B_PAGE_SIZE < info.max_pages / 2; + return uint64(fEditor.FileSize()) / B_PAGE_SIZE < info.max_pages / 2; } diff --git a/src/apps/packageinstaller/PackageInfo.cpp b/src/apps/packageinstaller/PackageInfo.cpp index 93549344ad..c0f212e46e 100644 --- a/src/apps/packageinstaller/PackageInfo.cpp +++ b/src/apps/packageinstaller/PackageInfo.cpp @@ -45,6 +45,21 @@ enum { P_SCRIPT }; +typedef enum { + B_BEBOX_PLATFORM = 0, + B_MAC_PLATFORM, + B_AT_CLONE_PLATFORM, + B_ENIAC_PLATFORM, + B_APPLE_II_PLATFORM, + B_CRAY_PLATFORM, + B_LISA_PLATFORM, + B_TI_994A_PLATFORM, + B_TIMEX_SINCLAIR_PLATFORM, + B_ORAC_1_PLATFORM, + B_HAL_PLATFORM, + B_INVALID_PLATFORM +} platform_type; + PackageInfo::PackageInfo() : @@ -131,8 +146,19 @@ PackageInfo::Parse() const char padding[7] = { 0, 0, 0, 0, 0, 0, 0 }; - system_info sysinfo; - get_system_info(&sysinfo); + platform_type thisPlatform = B_INVALID_PLATFORM; + cpu_topology_node_info topologyRoot; + uint32 topologyNodeCount = 1; + if (get_cpu_topology_info(&topologyRoot, &topologyNodeCount) == B_OK) { + switch (topologyRoot.data.root.platform) { + case B_CPU_x86: + thisPlatform = B_AT_CLONE_PLATFORM; + break; + + default: + break; + } + } uint64 infoOffset = 0, groupsOffset = 0; uint64 length = 0; @@ -843,8 +869,7 @@ PackageInfo::Parse() parser_debug("Padding!\n"); if (platform != 0xffffffff - && static_cast(platform) - != sysinfo.platform_type) { + && static_cast(platform) != thisPlatform) { // If the file/directory/item's platform is different than the // target platform (or different than the 'any' constant), // ignore this file diff --git a/src/apps/processcontroller/MemoryBarMenu.cpp b/src/apps/processcontroller/MemoryBarMenu.cpp index cdd8a925b4..125189dee0 100644 --- a/src/apps/processcontroller/MemoryBarMenu.cpp +++ b/src/apps/processcontroller/MemoryBarMenu.cpp @@ -45,7 +45,7 @@ MemoryBarMenu::MemoryBarMenu(const char* name, info_pack* infos, system_info& sy fTeamList = (team_id*)malloc(sizeof (team_id) * fTeamCount); - int k; + unsigned int k; for (k = 0; k < systemInfo.used_teams; k++) { fTeamList[k] = infos[k].team_info.team; } @@ -124,7 +124,7 @@ MemoryBarMenu::Pulse() info_pack infos; item = NULL; while (get_next_team_info(&cookie, &infos.team_info) == B_OK) { - int j = 0; + unsigned int j = 0; while (j < fTeamCount && infos.team_info.team != fTeamList[j]) { j++; } diff --git a/src/apps/processcontroller/MemoryBarMenu.h b/src/apps/processcontroller/MemoryBarMenu.h index 7f54961c41..627619608f 100644 --- a/src/apps/processcontroller/MemoryBarMenu.h +++ b/src/apps/processcontroller/MemoryBarMenu.h @@ -44,7 +44,7 @@ class MemoryBarMenu : public BMenu { private: team_id* fTeamList; - int fTeamCount; + unsigned int fTeamCount; MRecycleItem* fRecycleList; int fRecycleCount; bigtime_t fLastTotalTime; diff --git a/src/apps/processcontroller/ProcessController.cpp b/src/apps/processcontroller/ProcessController.cpp index 8e9765ea35..0e4a0d08b5 100644 --- a/src/apps/processcontroller/ProcessController.cpp +++ b/src/apps/processcontroller/ProcessController.cpp @@ -77,7 +77,7 @@ const rgb_color kKernelBlue = {20, 20, 231, 255}; const rgb_color kIdleGreen = {110, 190,110, 255}; ProcessController* gPCView; -int32 gCPUcount; +uint32 gCPUcount; rgb_color gUserColor; rgb_color gUserColorSelected; rgb_color gIdleColor; @@ -435,10 +435,10 @@ ProcessController::MessageReceived(BMessage *message) case 'CPU ': { - int32 cpu; - if (message->FindInt32 ("cpu", &cpu) == B_OK) { + uint32 cpu; + if (message->FindInt32("cpu", (int32*)&cpu) == B_OK) { bool last = true; - for (int p = 0; p < gCPUcount; p++) { + for (unsigned int p = 0; p < gCPUcount; p++) { if (p != cpu && _kern_cpu_enabled(p)) { last = false; break; @@ -609,7 +609,7 @@ ProcessController::DoDraw(bool force) SetHighColor(frame_color); StrokeRect(BRect(left - 1, top - 1, right, bottom + 1)); if (gCPUcount > 1 && layout[gCPUcount].cpu_inter == 1) { - for (int x = 1; x < gCPUcount; x++) + for (unsigned int x = 1; x < gCPUcount; x++) StrokeLine(BPoint(left + x * barWidth + x - 1, top), BPoint(left + x * barWidth + x - 1, bottom)); } @@ -619,7 +619,7 @@ ProcessController::DoDraw(bool force) StrokeRect(BRect(leftMem - 1, top - 1, leftMem + layout[gCPUcount].mem_width, bottom + 1)); - for (int x = 0; x < gCPUcount; x++) { + for (unsigned int x = 0; x < gCPUcount; x++) { right = left + barWidth - 1; float rem = fCPUTimes[x] * (h + 1); float barHeight = floorf (rem); @@ -697,14 +697,17 @@ ProcessController::Update() get_system_info(&info); bigtime_t now = system_time(); + cpu_info* cpuInfos = new cpu_info[gCPUcount]; + get_cpu_info(0, gCPUcount, cpuInfos); + fMemoryUsage = float(info.used_pages) / float(info.max_pages); // Calculate work done since last call to Update() for each CPU - for (int x = 0; x < gCPUcount; x++) { - bigtime_t load = info.cpu_infos[x].active_time - fPrevActive[x]; + for (unsigned int x = 0; x < gCPUcount; x++) { + bigtime_t load = cpuInfos[x].active_time - fPrevActive[x]; bigtime_t passed = now - fPrevTime; float cpuTime = float(load) / float(passed); - fPrevActive[x] = info.cpu_infos[x].active_time; + fPrevActive[x] = cpuInfos[x].active_time; if (load > passed) fPrevActive[x] -= load - passed; // save overload for next period... if (cpuTime < 0) @@ -714,6 +717,8 @@ ProcessController::Update() fCPUTimes[x] = cpuTime; } fPrevTime = now; + + delete[] cpuInfos; } @@ -725,7 +730,8 @@ thread_popup(void *arg) { Tpopup_param* param = (Tpopup_param*) arg; int32 mcookie, hcookie; - long m, h; + unsigned long m; + long h; BMenuItem* item; bool top = param->top; @@ -797,7 +803,7 @@ thread_popup(void *arg) // CPU on/off section if (gCPUcount > 1) { - for (int i = 0; i < gCPUcount; i++) { + for (unsigned int i = 0; i < gCPUcount; i++) { char item_name[32]; sprintf (item_name, B_TRANSLATE("Processor %d"), i + 1); BMessage* m = new BMessage ('CPU '); diff --git a/src/apps/processcontroller/ProcessController.h b/src/apps/processcontroller/ProcessController.h index 80c3a08c2b..8317d17747 100644 --- a/src/apps/processcontroller/ProcessController.h +++ b/src/apps/processcontroller/ProcessController.h @@ -73,7 +73,7 @@ class ProcessController : public BView { }; extern ProcessController* gPCView; -extern int32 gCPUcount; +extern uint32 gCPUcount; extern rgb_color gIdleColor; extern rgb_color gIdleColorSelected; extern rgb_color gKernelColor; diff --git a/src/apps/processcontroller/TeamBarMenuItem.cpp b/src/apps/processcontroller/TeamBarMenuItem.cpp index 3c9e5e477a..5806a6ebac 100644 --- a/src/apps/processcontroller/TeamBarMenuItem.cpp +++ b/src/apps/processcontroller/TeamBarMenuItem.cpp @@ -51,7 +51,7 @@ TeamBarMenuItem::Init() if (fTeamID == B_SYSTEM_TEAM) { thread_info thinfos; bigtime_t idle = 0; - for (int t = 1; t <= gCPUcount; t++) + for (unsigned int t = 1; t <= gCPUcount; t++) if (get_thread_info(t, &thinfos) == B_OK) idle += thinfos.kernel_time + thinfos.user_time; fTeamUsageInfo.kernel_time += fTeamUsageInfo.user_time; @@ -213,9 +213,10 @@ TeamBarMenuItem::BarUpdate() bigtime_t idle = 0; if (fTeamID == B_SYSTEM_TEAM) { thread_info thinfos; - for (int t = 1; t <= gCPUcount; t++) + for (unsigned int t = 1; t <= gCPUcount; t++) { if (get_thread_info(t, &thinfos) == B_OK) idle += thinfos.kernel_time + thinfos.user_time; + } usage.kernel_time += usage.user_time; usage.user_time = idle; idle -= fTeamUsageInfo.user_time; diff --git a/src/apps/pulse/DeskbarPulseView.cpp b/src/apps/pulse/DeskbarPulseView.cpp index 959c49daca..b20c3236f0 100644 --- a/src/apps/pulse/DeskbarPulseView.cpp +++ b/src/apps/pulse/DeskbarPulseView.cpp @@ -75,9 +75,8 @@ void DeskbarPulseView::AttachedToWindow() { system_info sys_info; get_system_info(&sys_info); if (sys_info.cpu_count >= 2) { - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) cpu_menu_items[x]->SetTarget(messenger); - } } // Use a BMessageRunner to deliver periodic messsages instead diff --git a/src/apps/pulse/MiniPulseView.cpp b/src/apps/pulse/MiniPulseView.cpp index 923810f439..32804892e2 100644 --- a/src/apps/pulse/MiniPulseView.cpp +++ b/src/apps/pulse/MiniPulseView.cpp @@ -76,7 +76,7 @@ void MiniPulseView::Draw(BRect rect) { float bar_width = (bounds.Width()) / sys_info.cpu_count - 2; float right = bar_width + left; - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) { int bar_height = (int)(cpu_times[x] * (h + 1)); if (bar_height > h) bar_height = h; double rem = cpu_times[x] * (h + 1) - bar_height; @@ -131,9 +131,8 @@ void MiniPulseView::AttachedToWindow() { system_info sys_info; get_system_info(&sys_info); if (sys_info.cpu_count >= 2) { - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) cpu_menu_items[x]->SetTarget(messenger); - } } } diff --git a/src/apps/pulse/NormalPulseView.cpp b/src/apps/pulse/NormalPulseView.cpp index 632b6bfa5c..c23dd24b69 100644 --- a/src/apps/pulse/NormalPulseView.cpp +++ b/src/apps/pulse/NormalPulseView.cpp @@ -129,16 +129,34 @@ NormalPulseView::DetermineVendorAndProcessor() logo = PowerPCLogo; #endif #if __INTEL__ - - switch (sys_info.cpu_type & B_CPU_x86_VENDOR_MASK) { - case B_CPU_INTEL_x86: - logo = IntelLogo; - break; - - case B_CPU_AMD_x86: - logo = AmdLogo; - break; - }; + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + + get_cpu_topology_info(NULL, &topologyNodeCount); + if (topologyNodeCount != 0) + topology = new cpu_topology_node_info[topologyNodeCount]; + get_cpu_topology_info(topology, &topologyNodeCount); + + for (uint32 i = 0; i < topologyNodeCount; i++) { + if (topology[i].type == B_TOPOLOGY_PACKAGE) { + switch (topology[i].data.package.vendor) { + case B_CPU_VENDOR_INTEL: + logo = IntelLogo; + break; + + case B_CPU_VENDOR_AMD: + logo = AmdLogo; + break; + + default: + break; + } + + break; + } + } + + delete[] topology; #endif fCpuLogo->SetBits(logo, fCpuLogo->BitsLength(), 0, B_CMAP8); @@ -255,9 +273,8 @@ NormalPulseView::AttachedToWindow() system_info sys_info; get_system_info(&sys_info); if (sys_info.cpu_count >= 2) { - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) cpu_menu_items[x]->SetTarget(messenger); - } } } @@ -270,7 +287,7 @@ NormalPulseView::UpdateColors(BMessage *message) system_info sys_info; get_system_info(&sys_info); - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) { fProgressBars[x]->UpdateColors(color, fade); fCpuButtons[x]->UpdateColors(color); } diff --git a/src/apps/pulse/PulseApp.cpp b/src/apps/pulse/PulseApp.cpp index 659762c2ba..af2f6f471b 100644 --- a/src/apps/pulse/PulseApp.cpp +++ b/src/apps/pulse/PulseApp.cpp @@ -207,14 +207,14 @@ PulseApp::ShowAbout(bool asApplication) */ bool -LastEnabledCPU(int my_cpu) +LastEnabledCPU(unsigned int my_cpu) { system_info sys_info; get_system_info(&sys_info); if (sys_info.cpu_count == 1) return true; - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) { if (x == my_cpu) continue; if (_kern_cpu_enabled(x) == 1) diff --git a/src/apps/pulse/PulseApp.h b/src/apps/pulse/PulseApp.h index d51cd3f344..266af30d4f 100644 --- a/src/apps/pulse/PulseApp.h +++ b/src/apps/pulse/PulseApp.h @@ -32,7 +32,7 @@ private: void BuildPulse(); }; -extern bool LastEnabledCPU(int cpu); +extern bool LastEnabledCPU(unsigned int cpu); extern int GetMinimumViewWidth(); extern bool LoadInDeskbar(); extern void Usage(); diff --git a/src/apps/pulse/PulseView.cpp b/src/apps/pulse/PulseView.cpp index 9370875a59..5a31421aac 100644 --- a/src/apps/pulse/PulseView.cpp +++ b/src/apps/pulse/PulseView.cpp @@ -82,7 +82,7 @@ void PulseView::Init() { if (sys_info.cpu_count >= 2) { cpu_menu_items = new BMenuItem *[sys_info.cpu_count]; char temp[20]; - for (int x = 0; x < sys_info.cpu_count; x++) { + for (unsigned int x = 0; x < sys_info.cpu_count; x++) { sprintf(temp, "CPU %d", x + 1); BMessage *message = new BMessage(PV_CPU_MENU_ITEM); message->AddInt32("which", x); @@ -115,10 +115,14 @@ void PulseView::Update() { get_system_info(&sys_info); bigtime_t now = system_time(); + cpu_info* cpuInfos = new cpu_info[sys_info.cpu_count]; + get_cpu_info(0, sys_info.cpu_count, cpuInfos); + // Calculate work done since last call to Update() for each CPU - for (int x = 0; x < sys_info.cpu_count; x++) { - double cpu_time = (double)(sys_info.cpu_infos[x].active_time - prev_active[x]) / (now - prev_time); - prev_active[x] = sys_info.cpu_infos[x].active_time; + for (unsigned int x = 0; x < sys_info.cpu_count; x++) { + double cpu_time = (double)(cpuInfos[x].active_time - prev_active[x]) + / (now - prev_time); + prev_active[x] = cpuInfos[x].active_time; if (cpu_time < 0) cpu_time = 0; if (cpu_time > 1) cpu_time = 1; cpu_times[x] = cpu_time; @@ -131,6 +135,8 @@ void PulseView::Update() { } } prev_time = now; + + delete[] cpuInfos; } void PulseView::ChangeCPUState(BMessage *message) { diff --git a/src/bin/Jamfile b/src/bin/Jamfile index 06abda94e3..756782d067 100644 --- a/src/bin/Jamfile +++ b/src/bin/Jamfile @@ -48,7 +48,6 @@ StdBinCommands release.c renice.c rescan.c - sysinfo.cpp unchop.c uptime.cpp vmstat.cpp @@ -159,6 +158,7 @@ StdBinCommands # commands that need libstdc++ only StdBinCommands diff_zip.cpp + sysinfo.cpp : $(TARGET_LIBSTDC++) : $(haiku-utils_rsrc) ; # standard commands that need libbe.so, libtranslation.so, libsupc++.so diff --git a/src/bin/coreutils/src/uname.c b/src/bin/coreutils/src/uname.c index 16b0b33c94..1bf091458c 100644 --- a/src/bin/coreutils/src/uname.c +++ b/src/bin/coreutils/src/uname.c @@ -346,16 +346,20 @@ main (int argc, char **argv) #ifdef __HAIKU__ { - system_info sysinfo; - get_system_info(&sysinfo); - - switch (sysinfo.platform_type) { - case B_AT_CLONE_PLATFORM: - element = "x86"; - break; - case B_64_BIT_PC_PLATFORM: - element = "x86_64"; - break; + cpu_topology_node_info root; + uint32_t count = 1; + status_t error = get_cpu_topology_info(&root, &count); + if (error != B_OK || count < 1) + element = "unknown"; + else { + switch (root.data.root.platform) { + case B_CPU_x86: + element = "x86"; + break; + case B_CPU_x86_64: + element = "x86_64"; + break; + } } } #endif diff --git a/src/bin/sysinfo.cpp b/src/bin/sysinfo.cpp index 552b3b4f72..02dee0ec84 100644 --- a/src/bin/sysinfo.cpp +++ b/src/bin/sysinfo.cpp @@ -141,7 +141,8 @@ struct cache_description { static void -print_intel_cache_descriptors(enum cpu_types type, cpuid_info *info) +print_intel_cache_descriptors(enum cpu_vendor vendor, uint32 model, + cpuid_info *info) { uint8 cacheDescriptors[15]; // Max @@ -195,9 +196,8 @@ print_intel_cache_descriptors(enum cpu_types type, cpuid_info *info) if (cacheDescriptors[i] == sIntelCacheDescriptions[j].code) { if (cacheDescriptors[i] == 0x40) { printf("\tNo integrated L%u cache\n", - type >= B_CPU_INTEL_PENTIUM_IV - && (type & B_CPU_x86_VENDOR_MASK) == B_CPU_INTEL_x86 - ? 3 : 2); + ((model >> 8) & 0xf) == 0xf + && vendor == B_CPU_VENDOR_INTEL ? 3 : 2); } else printf("\t%s\n", sIntelCacheDescriptions[j].description); break; @@ -210,6 +210,7 @@ print_intel_cache_descriptors(enum cpu_types type, cpuid_info *info) } } + #endif // __INTEL__ || __x86_64__ @@ -466,12 +467,11 @@ print_features(uint32 features) #if defined(__INTEL__) || defined(__x86_64__) static void -print_processor_signature(system_info *sys_info, cpuid_info *info, +print_processor_signature(enum cpu_vendor vendor, cpuid_info *info, const char *prefix) { - if ((sys_info->cpu_type & B_CPU_x86_VENDOR_MASK) == B_CPU_AMD_x86) { - + if (vendor == B_CPU_VENDOR_AMD) { printf("\t%s%sype %" B_PRIu32 ", family %" B_PRIu32 ", model %" B_PRIu32 ", stepping %" B_PRIu32 ", features 0x%08" B_PRIx32 "\n", prefix ? prefix : "", prefix && prefix[0] ? "t" : "T", @@ -482,8 +482,7 @@ print_processor_signature(system_info *sys_info, cpuid_info *info, ? info->eax_1.extended_model << 4 : 0), info->eax_1.stepping, info->eax_1.features); - } else if ((sys_info->cpu_type & B_CPU_x86_VENDOR_MASK) - == B_CPU_INTEL_x86) { + } else if (vendor == B_CPU_VENDOR_INTEL) { // model calculation is different for INTEL printf("\t%s%sype %" B_PRIu32 ", family %" B_PRIu32 ", model %" B_PRIu32 ", stepping %" B_PRIu32 ", features 0x%08" B_PRIx32 "\n", @@ -505,17 +504,33 @@ print_processor_signature(system_info *sys_info, cpuid_info *info, static void dump_platform(system_info *info) { - printf("%s\n", - info->platform_type == B_AT_CLONE_PLATFORM ? "IntelArchitecture" : - info->platform_type == B_MAC_PLATFORM ? "Macintosh" : - info->platform_type == B_BEBOX_PLATFORM ? "BeBox" : "unknown"); + cpu_topology_node_info root; + uint32 count = 1; + get_cpu_topology_info(&root, &count); + + const char* platform; + switch (root.data.root.platform) { + case B_CPU_x86: + platform = "IntelArchitecture"; + break; + + case B_CPU_x86_64: + platform = "IntelArchitecture (64 bit)"; + break; + + default: + platform = "unknown"; + break; + } + + printf("%s\n", platform); } #if defined(__INTEL__) || defined(__x86_64__) static void -dump_cpu(system_info *info, int32 cpu) +dump_cpu(enum cpu_vendor vendor, uint32 model, int32 cpu) { // References: // http://grafi.ii.pw.edu.pl/gbm/x86/cpuid.html @@ -573,7 +588,7 @@ dump_cpu(system_info *info, int32 cpu) else { // Intel CPUs don't seem to have the genuine vendor field printf("CPU #%" B_PRId32 ": %.12s\n", cpu, - (info->cpu_type & B_CPU_x86_VENDOR_MASK) == B_CPU_INTEL_x86 ? + vendor == B_CPU_VENDOR_INTEL ? baseInfo.eax_0.vendor_id : cpuInfo.eax_0.vendor_id); } } else { @@ -583,7 +598,7 @@ dump_cpu(system_info *info, int32 cpu) } get_cpuid(&cpuInfo, 1, cpu); - print_processor_signature(info, &cpuInfo, NULL); + print_processor_signature(vendor, &cpuInfo, NULL); print_features(cpuInfo.eax_1.features); if (maxStandardFunction >= 1) { @@ -595,17 +610,15 @@ dump_cpu(system_info *info, int32 cpu) /* Extended CPUID */ if (maxExtendedFunction >= 1) { get_cpuid(&cpuInfo, 0x80000001, cpu); - print_processor_signature(info, &cpuInfo, "Extended AMD: "); + print_processor_signature(vendor, &cpuInfo, "Extended AMD: "); - if ((info->cpu_type & B_CPU_x86_VENDOR_MASK) == B_CPU_AMD_x86 - || (info->cpu_type & B_CPU_x86_VENDOR_MASK) == B_CPU_INTEL_x86) { + if (vendor == B_CPU_VENDOR_AMD || vendor == B_CPU_VENDOR_INTEL) { print_amd_features(cpuInfo.regs.edx); if (maxExtendedFunction >= 7) { get_cpuid(&cpuInfo, 0x80000007, cpu); print_amd_power_management_features(cpuInfo.regs.edx); } - } else if ((info->cpu_type & B_CPU_x86_VENDOR_MASK) - == B_CPU_TRANSMETA_x86) + } else if (vendor == B_CPU_VENDOR_TRANSMETA) print_transmeta_features(cpuInfo.regs.edx); } @@ -613,9 +626,8 @@ dump_cpu(system_info *info, int32 cpu) if (maxExtendedFunction >= 5) { if (!strncmp(baseInfo.eax_0.vendor_id, "CyrixInstead", 12)) { get_cpuid(&cpuInfo, 0x00000002, cpu); - print_intel_cache_descriptors(info->cpu_type, &cpuInfo); - } else if ((info->cpu_type & B_CPU_x86_VENDOR_MASK) - == B_CPU_INTEL_x86) { + print_intel_cache_descriptors(vendor, model, &cpuInfo); + } else if (vendor == B_CPU_VENDOR_INTEL) { // Intel does not support extended function 5 (but it does 6 hmm) print_intel_cache_desc(cpu); } else { @@ -628,7 +640,7 @@ dump_cpu(system_info *info, int32 cpu) get_cpuid(&cpuInfo, 2, cpu); if (cpuInfo.eax_2.call_num > 0) - print_intel_cache_descriptors(info->cpu_type, &cpuInfo); + print_intel_cache_descriptors(vendor, model, &cpuInfo); } while (cpuInfo.eax_2.call_num > 1); } @@ -657,28 +669,59 @@ dump_cpu(system_info *info, int32 cpu) static void dump_cpus(system_info *info) { - const char *vendor = get_cpu_vendor_string(info->cpu_type); - const char *model = get_cpu_model_string(info); + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + get_cpu_topology_info(NULL, &topologyNodeCount); + if (topologyNodeCount != 0) + topology = new cpu_topology_node_info[topologyNodeCount]; + get_cpu_topology_info(topology, &topologyNodeCount); + + enum cpu_platform platform = B_CPU_UNKNOWN; + enum cpu_vendor cpuVendor = B_CPU_VENDOR_UNKNOWN; + uint32 cpuModel = 0; + uint64 frequency = 0; + for (uint32 i = 0; i < topologyNodeCount; i++) { + switch (topology[i].type) { + case B_TOPOLOGY_ROOT: + platform = topology[i].data.root.platform; + break; + + case B_TOPOLOGY_PACKAGE: + cpuVendor = topology[i].data.package.vendor; + break; + + case B_TOPOLOGY_CORE: + cpuModel = topology[i].data.core.model; + frequency = topology[i].data.core.default_frequency; + break; + + default: + break; + } + } + delete[] topology; + + const char *vendor = get_cpu_vendor_string(cpuVendor); + const char *model = get_cpu_model_string(platform, cpuVendor, cpuModel); char modelString[32]; if (model == NULL && vendor == NULL) model = "(Unknown)"; else if (model == NULL) { model = modelString; - snprintf(modelString, 32, "(Unknown %x)", info->cpu_type); + snprintf(modelString, 32, "(Unknown %" B_PRIx32 ")", cpuModel); } printf("%" B_PRId32 " %s%s%s, revision %04" B_PRIx32 " running at %" - B_PRId64 "MHz (ID: 0x%08" B_PRIx32 " 0x%08" B_PRIx32 ")\n\n", + B_PRIu64 "MHz\n\n", info->cpu_count, vendor ? vendor : "", vendor ? " " : "", model, - info->cpu_revision, - info->cpu_clock_speed / 1000000, - info->id[0], info->id[1]); + cpuModel, + frequency / 1000000); #if defined(__INTEL__) || defined(__x86_64__) - for (int32 cpu = 0; cpu < info->cpu_count; cpu++) - dump_cpu(info, cpu); + for (uint32 cpu = 0; cpu < info->cpu_count; cpu++) + dump_cpu(cpuVendor, cpuModel, cpu); #endif // __INTEL__ || __x86_64__ } @@ -780,8 +823,7 @@ main(int argc, char *argv[]) const char *opt = argv[i]; if (strncmp(opt, "-id", strlen(opt)) == 0) { /* note: the original also assumes this option on "sysinfo -" */ - printf("0x%.8" B_PRIx32 " 0x%.8" B_PRIx32 "\n", info.id[0], - info.id[1]); + printf("%#.8x %#.8x\n", 0,0); } else if (strncmp(opt, "-cpu", strlen(opt)) == 0) { dump_cpus(&info); } else if (strncmp(opt, "-mem", strlen(opt)) == 0) { diff --git a/src/kits/tracker/TaskLoop.cpp b/src/kits/tracker/TaskLoop.cpp index 6a3b00ed95..c87fdc310f 100644 --- a/src/kits/tracker/TaskLoop.cpp +++ b/src/kits/tracker/TaskLoop.cpp @@ -172,8 +172,14 @@ ActivityLevel() bigtime_t time = 0; system_info sinfo; get_system_info(&sinfo); - for (int32 index = 0; index < sinfo.cpu_count; index++) - time += sinfo.cpu_infos[index].active_time; + + cpu_info* cpuInfos = new cpu_info[sinfo.cpu_count]; + get_cpu_info(0, sinfo.cpu_count, cpuInfos); + + for (uint32 index = 0; index < sinfo.cpu_count; index++) + time += cpuInfos[index].active_time; + + delete[] cpuInfos; return time / ((bigtime_t) sinfo.cpu_count); } diff --git a/src/servers/app/drawing/Painter/Painter.cpp b/src/servers/app/drawing/Painter/Painter.cpp index 2dc6bb884c..3654a8b942 100644 --- a/src/servers/app/drawing/Painter/Painter.cpp +++ b/src/servers/app/drawing/Painter/Painter.cpp @@ -125,7 +125,7 @@ detect_simd() // supported across all CPUs found. uint32 systemSIMD = 0xffffffff; - for (int32 cpu = 0; cpu < systemInfo.cpu_count; cpu++) { + for (uint32 cpu = 0; cpu < systemInfo.cpu_count; cpu++) { cpuid_info cpuInfo; get_cpuid(&cpuInfo, 0, cpu); diff --git a/src/system/kernel/arch/x86/arch_system_info.cpp b/src/system/kernel/arch/x86/arch_system_info.cpp index 280ca0eb65..0544c8105b 100644 --- a/src/system/kernel/arch/x86/arch_system_info.cpp +++ b/src/system/kernel/arch/x86/arch_system_info.cpp @@ -17,9 +17,9 @@ #include -uint32 sCpuType; -int32 sCpuRevision; -int64 sCpuClockSpeed; +enum cpu_vendor sCPUVendor; +uint32 sCPUModel; +int64 sCPUClockSpeed; static bool @@ -60,87 +60,74 @@ get_cpuid(cpuid_info *info, uint32 eaxRegister, uint32 forCPU) status_t -arch_get_system_info(system_info *info, size_t size) +arch_system_info_init(struct kernel_args *args) { - info->cpu_type = (cpu_types)sCpuType; - info->cpu_revision = sCpuRevision; + // So far we don't have to care about heterogeneous x86 platforms. + cpu_ent* cpu = get_cpu_struct(); - // - various cpu_info - info->cpu_clock_speed = sCpuClockSpeed; - // - bus_clock_speed -#ifdef __x86_64__ - info->platform_type = B_64_BIT_PC_PLATFORM; -#else - info->platform_type = B_AT_CLONE_PLATFORM; -#endif + switch (cpu->arch.vendor) { + case VENDOR_AMD: + sCPUVendor = B_CPU_VENDOR_AMD; + break; + case VENDOR_CENTAUR: + sCPUVendor = B_CPU_VENDOR_VIA; + break; + case VENDOR_CYRIX: + sCPUVendor = B_CPU_VENDOR_CYRIX; + break; + case VENDOR_INTEL: + sCPUVendor = B_CPU_VENDOR_INTEL; + break; + case VENDOR_NSC: + sCPUVendor = B_CPU_VENDOR_NATIONAL_SEMICONDUCTOR; + break; + case VENDOR_RISE: + sCPUVendor = B_CPU_VENDOR_RISE; + break; + case VENDOR_TRANSMETA: + sCPUVendor = B_CPU_VENDOR_TRANSMETA; + break; + default: + sCPUVendor = B_CPU_VENDOR_UNKNOWN; + break; + } - // ToDo: clock speeds could be retrieved via SMBIOS/DMI + sCPUModel = (cpu->arch.extended_family << 20) + | (cpu->arch.extended_model << 16) | (cpu->arch.type << 12) + | (cpu->arch.family << 8) | (cpu->arch.model << 4) | cpu->arch.stepping; + + sCPUClockSpeed = args->arch_args.cpu_clock_speed; return B_OK; } -status_t -arch_system_info_init(struct kernel_args *args) +void +arch_fill_topology_node(cpu_topology_node_info* node, int32 cpu) { - // This is what you get if the CPU vendor is not recognized - // or the CPU does not support cpuid with eax == 1. - uint32 base; - uint32 model = 0; - cpu_ent *cpu = get_cpu_struct(); + switch (node->type) { + case B_TOPOLOGY_ROOT: +#if __INTEL__ + node->data.root.platform = B_CPU_x86; +#elif __x86_64__ + node->data.root.platform = B_CPU_x86_64; +#else + node->data.root.platform = B_CPU_UNKNOWN; +#endif + break; - switch (cpu->arch.vendor) { - case VENDOR_INTEL: - base = B_CPU_INTEL_x86; + case B_TOPOLOGY_PACKAGE: + node->data.package.vendor = sCPUVendor; + node->data.package.cache_line_size = CACHE_LINE_SIZE; break; - case VENDOR_AMD: - base = B_CPU_AMD_x86; - break; - case VENDOR_CYRIX: - base = B_CPU_CYRIX_x86; - break; - case VENDOR_UMC: - base = B_CPU_INTEL_x86; // XXX - break; - case VENDOR_NEXGEN: - base = B_CPU_INTEL_x86; // XXX - break; - case VENDOR_CENTAUR: - base = B_CPU_VIA_IDT_x86; - break; - case VENDOR_RISE: - base = B_CPU_RISE_x86; - break; - case VENDOR_TRANSMETA: - base = B_CPU_TRANSMETA_x86; - break; - case VENDOR_NSC: - base = B_CPU_NATIONAL_x86; + + case B_TOPOLOGY_CORE: + node->data.core.model = sCPUModel; + node->data.core.default_frequency = sCPUClockSpeed; break; + default: - base = B_CPU_x86; + break; } - - if (base != B_CPU_x86) { - if (base == B_CPU_INTEL_x86 - || (base == B_CPU_AMD_x86 && cpu->arch.family == 0xF)) { - model = (cpu->arch.extended_family << 20) - + (cpu->arch.extended_model << 16) - + (cpu->arch.family << 4) + cpu->arch.model; - } else { - model = (cpu->arch.family << 4) - + cpu->arch.model; - // Isn't much useful extended family and model information - // yet on other processors. - } - } - - sCpuRevision = (cpu->arch.extended_family << 18) - | (cpu->arch.extended_model << 14) | (cpu->arch.type << 12) - | (cpu->arch.family << 8) | (cpu->arch.model << 4) | cpu->arch.stepping; - - sCpuType = base + model; - sCpuClockSpeed = args->arch_args.cpu_clock_speed; - return B_OK; } @@ -164,3 +151,4 @@ _user_get_cpuid(cpuid_info *userInfo, uint32 eaxRegister, uint32 cpuNum) return status; } + diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index d6713b00c0..709fe30860 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -1,10 +1,11 @@ /* - * Copyright (c) 2004-2010, Haiku, Inc. + * Copyright (c) 2004-2013, Haiku, Inc. * Distributed under the terms of the MIT license. * * Authors: * Stefano Ceccherini * Axel Dörfler, axeld@pinc-software.de + * Paweł Dziepak, pdziepak@quarnos.org * Ingo Weinhold, ingo_weinhold@gmx.de */ @@ -16,6 +17,8 @@ #include +#include + #include #include @@ -407,21 +410,14 @@ SystemNotificationService::Listener::OwnerDeleted(AssociatedDataOwner* owner) status_t -_get_system_info(system_info *info, size_t size) +get_system_info(system_info* info) { - if (size != sizeof(system_info)) - return B_BAD_VALUE; - memset(info, 0, sizeof(system_info)); info->boot_time = rtc_boot_time(); info->cpu_count = smp_get_num_cpus(); - for (int32 i = 0; i < info->cpu_count; i++) - info->cpu_infos[i].active_time = cpu_get_active_time(i); - vm_page_get_stats(info); - // TODO: Add page_faults info->used_threads = thread_used_threads(); info->max_threads = thread_max_threads(); @@ -432,17 +428,40 @@ _get_system_info(system_info *info, size_t size) info->used_sems = sem_used_sems(); info->max_sems = sem_max_sems(); + // TODO: fill the new fields + info->kernel_version = kKernelVersion; strlcpy(info->kernel_name, kKernelName, B_FILE_NAME_LENGTH); strlcpy(info->kernel_build_date, __DATE__, B_OS_NAME_LENGTH); strlcpy(info->kernel_build_time, __TIME__, B_OS_NAME_LENGTH); info->abi = B_HAIKU_ABI; - // all other stuff is architecture specific - return arch_get_system_info(info, size); + return B_OK; } +status_t +get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info) +{ + if (firstCPU >= (uint32)smp_get_num_cpus()) + return B_BAD_VALUE; + if (cpuCount == 0) + return B_OK; + + uint32 count = std::min(cpuCount, smp_get_num_cpus() - firstCPU); + + memset(info, 0, sizeof(cpu_info) * count); + for (uint32 i = 0; i < count; i++) { + info[i].active_time = cpu_get_active_time(firstCPU + i); + // TODO: cpu_info::load + info[i].enabled = !gCPU[firstCPU + i].disabled; + } + + return B_OK; +} + + + status_t system_info_init(struct kernel_args *args) { @@ -471,15 +490,13 @@ system_notifications_init() status_t -_user_get_system_info(system_info *userInfo, size_t size) +_user_get_system_info(system_info* userInfo) { - // The BeBook says get_system_info() always returns B_OK, - // but that ain't true with invalid addresses if (userInfo == NULL || !IS_USER_ADDRESS(userInfo)) return B_BAD_ADDRESS; system_info info; - status_t status = _get_system_info(&info, size); + status_t status = get_system_info(&info); if (status == B_OK) { if (user_memcpy(userInfo, &info, sizeof(system_info)) < B_OK) return B_BAD_ADDRESS; @@ -491,6 +508,123 @@ _user_get_system_info(system_info *userInfo, size_t size) } +status_t +_user_get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* userInfo) +{ + if (userInfo == NULL || !IS_USER_ADDRESS(userInfo)) + return B_BAD_ADDRESS; + if (firstCPU >= (uint32)smp_get_num_cpus()) + return B_BAD_VALUE; + if (cpuCount == 0) + return B_OK; + + uint32 count = std::min(cpuCount, smp_get_num_cpus() - firstCPU); + + cpu_info* cpuInfos = new(std::nothrow) cpu_info[count]; + if (cpuInfos == NULL) + return B_NO_MEMORY; + ArrayDeleter _(cpuInfos); + + status_t error = get_cpu_info(firstCPU, count, cpuInfos); + if (error != B_OK) + return error; + + return user_memcpy(userInfo, cpuInfos, sizeof(cpu_info) * count); +} + + +static void +count_topology_nodes(const cpu_topology_node* node, uint32& count) +{ + count++; + for (int32 i = 0; i < node->children_count; i++) + count_topology_nodes(node->children[i], count); +} + + +static int32 +get_logical_processor(const cpu_topology_node* node) +{ + while (node->level != CPU_TOPOLOGY_SMT) { + ASSERT(node->children_count > 0); + node = node->children[0]; + } + + return node->id; +} + + +static cpu_topology_node_info* +generate_topology_array(cpu_topology_node_info* topology, + const cpu_topology_node* node, uint32& count) +{ + if (count == 0) + return topology; + + static const topology_level_type mapTopologyLevels[] = { B_TOPOLOGY_SMT, + B_TOPOLOGY_CORE, B_TOPOLOGY_PACKAGE, B_TOPOLOGY_ROOT }; + + STATIC_ASSERT(sizeof(mapTopologyLevels) / sizeof(topology_level_type) + == CPU_TOPOLOGY_LEVELS + 1); + + topology->id = node->id; + topology->level = node->level; + topology->type = mapTopologyLevels[node->level]; + + arch_fill_topology_node(topology, get_logical_processor(node)); + + count--; + topology++; + for (int32 i = 0; i < node->children_count && count > 0; i++, count--) + topology = generate_topology_array(topology, node->children[i], count); + return topology; +} + + +status_t +_user_get_cpu_topology_info(cpu_topology_node_info* topologyInfos, + uint32* topologyInfoCount) +{ + if (topologyInfoCount == NULL || !IS_USER_ADDRESS(topologyInfoCount)) + return B_BAD_ADDRESS; + + const cpu_topology_node* node = get_cpu_topology(); + + uint32 count = 0; + count_topology_nodes(node, count); + + if (topologyInfos == NULL) + return user_memcpy(topologyInfoCount, &count, sizeof(uint32)); + else if (!IS_USER_ADDRESS(topologyInfoCount)) + return B_BAD_ADDRESS; + + uint32 userCount; + status_t error = user_memcpy(&userCount, topologyInfoCount, sizeof(uint32)); + if (error != B_OK) + return error; + if (userCount == 0) + return B_OK; + count = std::min(count, userCount); + + cpu_topology_node_info* topology + = new(std::nothrow) cpu_topology_node_info[count]; + if (topology == NULL) + return B_NO_MEMORY; + ArrayDeleter _(topology); + memset(topology, 0, sizeof(cpu_topology_node_info) * count); + + uint32 nodesLeft = count; + generate_topology_array(topology, node, nodesLeft); + ASSERT(nodesLeft == 0); + + error = user_memcpy(topologyInfos, topology, + sizeof(cpu_topology_node_info) * count); + if (error != B_OK) + return error; + return user_memcpy(topologyInfoCount, &count, sizeof(uint32)); +} + + status_t _user_get_system_info_etc(int32 id, void* userInfo, size_t size) { diff --git a/src/system/libroot/os/Jamfile b/src/system/libroot/os/Jamfile index ca427584b4..193440898c 100644 --- a/src/system/libroot/os/Jamfile +++ b/src/system/libroot/os/Jamfile @@ -31,7 +31,7 @@ for architectureObject in [ MultiArchSubDirSetup ] { port.c scheduler.c sem.c - system_info.c + system_info.cpp system_revision.c team.c thread.c diff --git a/src/system/libroot/os/system_info.c b/src/system/libroot/os/system_info.c deleted file mode 100644 index 0392337fda..0000000000 --- a/src/system/libroot/os/system_info.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de. - * Distributed under the terms of the MIT License. - */ - - -#include - -#include -#include - - -status_t -_get_system_info(system_info *info, size_t size) -{ - if (info == NULL || size != sizeof(system_info)) - return B_BAD_VALUE; - - return _kern_get_system_info(info, size); -} - - -status_t -__get_system_info_etc(int32 id, void *info, size_t size) -{ - if (info == NULL || size == 0 || id < 0) - return B_BAD_VALUE; - - return _kern_get_system_info_etc(id, info, size); -} - - -status_t -__start_watching_system(int32 object, uint32 flags, port_id port, int32 token) -{ - return _kern_start_watching_system(object, flags, port, token); -} - - -status_t -__stop_watching_system(int32 object, uint32 flags, port_id port, int32 token) -{ - return _kern_stop_watching_system(object, flags, port, token); -} - - -int32 -is_computer_on(void) -{ - return _kern_is_computer_on(); -} - - -double -is_computer_on_fire(void) -{ - return 0.63739; -} - diff --git a/src/system/libroot/os/system_info.cpp b/src/system/libroot/os/system_info.cpp new file mode 100644 index 0000000000..adf7a0c710 --- /dev/null +++ b/src/system/libroot/os/system_info.cpp @@ -0,0 +1,242 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Copyright 2002-2008, Axel Dörfler, axeld@pinc-software.de. + * Distributed under the terms of the MIT License. + */ + + +#include + +#include + +#include + +#include +#include + + +#if _BEOS_R5_COMPATIBLE_ + + +enum cpu_type { + B_CPU_PPC_601 = 1, + B_CPU_PPC_603 = 2, + B_CPU_PPC_603e = 3, + B_CPU_PPC_604 = 4, + B_CPU_PPC_604e = 5, + B_CPU_PPC_686 = 13, + B_CPU_AMD_29K, + B_CPU_X86, + B_CPU_MC6502, + B_CPU_Z80, + B_CPU_ALPHA, + B_CPU_MIPS, + B_CPU_HPPA, + B_CPU_M68K, + B_CPU_ARM, + B_CPU_SH, + B_CPU_SPARC +}; + +enum platform_type { + B_BEBOX_PLATFORM = 0, + B_MAC_PLATFORM, + B_AT_CLONE_PLATFORM, + B_ENIAC_PLATFORM, + B_APPLE_II_PLATFORM, + B_CRAY_PLATFORM, + B_LISA_PLATFORM, + B_TI_994A_PLATFORM, + B_TIMEX_SINCLAIR_PLATFORM, + B_ORAC_1_PLATFORM, + B_HAL_PLATFORM +}; + +typedef struct { + bigtime_t active_time; /* usec of doing useful work since boot */ +} legacy_cpu_info; + +typedef struct { + int32 id[2]; /* unique machine ID */ + bigtime_t boot_time; /* time of boot (usecs since 1/1/1970) */ + + int32 cpu_count; /* number of cpus */ + enum cpu_type cpu_type; /* type of cpu */ + int32 cpu_revision; /* revision # of cpu */ + legacy_cpu_info cpu_infos[8]; /* info about individual cpus */ + int64 cpu_clock_speed; /* processor clock speed (Hz) */ + int64 bus_clock_speed; /* bus clock speed (Hz) */ + enum platform_type platform_type; /* type of machine we're on */ + + int32 max_pages; /* total # of accessible pages */ + int32 used_pages; /* # of accessible pages in use */ + int32 page_faults; /* # of page faults */ + int32 max_sems; + int32 used_sems; + int32 max_ports; + int32 used_ports; + int32 max_threads; + int32 used_threads; + int32 max_teams; + int32 used_teams; + + char kernel_name[256]; + char kernel_build_date[32]; + char kernel_build_time[32]; + int64 kernel_version; + + bigtime_t _busy_wait_time; /* reserved for whatever */ + + int32 cached_pages; + uint32 abi; /* the system API */ + int32 ignored_pages; /* # of ignored/inaccessible pages */ + int32 pad; +} legacy_system_info; + + +extern "C" status_t +_get_system_info(legacy_system_info* info, size_t size) +{ + if (info == NULL || size != sizeof(legacy_system_info)) + return B_BAD_VALUE; + memset(info, 0, sizeof(legacy_system_info)); + + system_info systemInfo; + status_t error = _kern_get_system_info(&systemInfo); + if (error != B_OK) + return error; + + cpu_info cpuInfos[8]; + error = _kern_get_cpu_info(0, std::min(systemInfo.cpu_count, uint32(8)), + cpuInfos); + if (error != B_OK) + return error; + + info->boot_time = systemInfo.boot_time; + info->cpu_count = std::min(systemInfo.cpu_count, uint32(8)); + for (int32 i = 0; i < info->cpu_count; i++) + info->cpu_infos[i].active_time = cpuInfos[i].active_time; + + info->platform_type = B_AT_CLONE_PLATFORM; + info->cpu_type = B_CPU_X86; + + uint32 topologyNodeCount = 0; + cpu_topology_node_info* topology = NULL; + error = get_cpu_topology_info(NULL, &topologyNodeCount); + if (error != B_OK) + return B_OK; + if (topologyNodeCount != 0) { + topology = new(std::nothrow) cpu_topology_node_info[topologyNodeCount]; + if (topology == NULL) + return B_NO_MEMORY; + } + error = get_cpu_topology_info(topology, &topologyNodeCount); + if (error != B_OK) { + delete[] topology; + return error; + } + + for (uint32 i = 0; i < topologyNodeCount; i++) { + if (topology[i].type == B_TOPOLOGY_CORE) { + info->cpu_clock_speed = topology[i].data.core.default_frequency; + break; + } + } + info->bus_clock_speed = info->cpu_clock_speed; + delete[] topology; + + info->max_pages = std::min(systemInfo.max_pages, uint64(INT32_MAX)); + info->used_pages = std::min(systemInfo.used_pages, uint64(INT32_MAX)); + info->cached_pages = std::min(systemInfo.cached_pages, uint64(INT32_MAX)); + info->ignored_pages = std::min(systemInfo.ignored_pages, uint64(INT32_MAX)); + info->page_faults = std::min(systemInfo.page_faults, uint32(INT32_MAX)); + info->max_sems = std::min(systemInfo.max_sems, uint32(INT32_MAX)); + info->used_sems = std::min(systemInfo.used_sems, uint32(INT32_MAX)); + info->max_ports = std::min(systemInfo.max_ports, uint32(INT32_MAX)); + info->used_ports = std::min(systemInfo.used_ports, uint32(INT32_MAX)); + info->max_threads = std::min(systemInfo.max_threads, uint32(INT32_MAX)); + info->used_threads = std::min(systemInfo.used_threads, uint32(INT32_MAX)); + info->max_teams = std::min(systemInfo.max_teams, uint32(INT32_MAX)); + info->used_teams = std::min(systemInfo.used_teams, uint32(INT32_MAX)); + + strlcpy(info->kernel_name, systemInfo.kernel_name, + sizeof(info->kernel_name)); + strlcpy(info->kernel_build_date, systemInfo.kernel_build_date, + sizeof(info->kernel_build_date)); + strlcpy(info->kernel_build_time, systemInfo.kernel_build_time, + sizeof(info->kernel_build_time)); + info->kernel_version = systemInfo.kernel_version; + + info->abi = systemInfo.abi; + + return B_OK; +} + + +#endif + + +status_t +__get_system_info(system_info* info) +{ + return _kern_get_system_info(info); +} + + +status_t +__get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info) +{ + return _kern_get_cpu_info(firstCPU, cpuCount, info); +} + + +status_t +__get_cpu_topology_info(cpu_topology_node_info* topologyInfos, + uint32* topologyInfoCount) +{ + return _kern_get_cpu_topology_info(topologyInfos, topologyInfoCount); +} + + +status_t +__get_system_info_etc(int32 id, void *info, size_t size) +{ + if (info == NULL || size == 0 || id < 0) + return B_BAD_VALUE; + + return _kern_get_system_info_etc(id, info, size); +} + + +status_t +__start_watching_system(int32 object, uint32 flags, port_id port, int32 token) +{ + return _kern_start_watching_system(object, flags, port, token); +} + + +status_t +__stop_watching_system(int32 object, uint32 flags, port_id port, int32 token) +{ + return _kern_stop_watching_system(object, flags, port, token); +} + + +int32 +is_computer_on(void) +{ + return _kern_is_computer_on(); +} + + +double +is_computer_on_fire(void) +{ + return 0.63739; +} + + +B_DEFINE_WEAK_ALIAS(__get_system_info, get_system_info); +B_DEFINE_WEAK_ALIAS(__get_cpu_info, get_cpu_info); +B_DEFINE_WEAK_ALIAS(__get_cpu_topology_info, get_cpu_topology_info); + diff --git a/src/system/libroot/posix/sys/uname.c b/src/system/libroot/posix/sys/uname.c index 120a95157d..18ea62a07a 100644 --- a/src/system/libroot/posix/sys/uname.c +++ b/src/system/libroot/posix/sys/uname.c @@ -20,9 +20,12 @@ int uname(struct utsname *info) { + cpu_topology_node_info root; system_info systemInfo; const char *platform; const char *haikuRevision; + uint32_t count = 1; + status_t error; if (!info) { __set_errno(B_BAD_VALUE); @@ -44,24 +47,20 @@ uname(struct utsname *info) snprintf(info->release, sizeof(info->release), "%" B_PRId64, systemInfo.kernel_version); - // TODO: make this better - switch (systemInfo.platform_type) { - case B_BEBOX_PLATFORM: - platform = "BeBox"; - break; - case B_MAC_PLATFORM: - platform = "BeMac"; - break; - case B_AT_CLONE_PLATFORM: - platform = "BePC"; - break; - case B_64_BIT_PC_PLATFORM: - platform = "x86_64"; - break; - default: - platform = "unknown"; - break; + error = get_cpu_topology_info(&root, &count); + if (error != B_OK || count < 1) + platform = "unknown"; + else { + switch (root.data.root.platform) { + case B_CPU_x86: + platform = "BePC"; + break; + case B_CPU_x86_64: + platform = "x86_64"; + break; + } } + strlcpy(info->machine, platform, sizeof(info->machine)); if (gethostname(info->nodename, sizeof(info->nodename)) != 0) From d02aaee17e007631fcfa91a012ec7b6386927012 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 16 Dec 2013 04:53:46 +0100 Subject: [PATCH 197/273] kernel, libroot: Add more memory info in system_info system_info now contains all information previously available only through __get_system_info_etc(B_MEMORY_INFO, ...). --- headers/os/kernel/OS.h | 7 ++-- headers/private/kernel/vm/vm.h | 2 +- headers/private/system/syscalls.h | 2 - headers/private/system/system_info.h | 16 -------- src/apps/activitymonitor/DataSource.cpp | 8 +--- src/apps/activitymonitor/SystemInfo.cpp | 22 +++++++---- src/apps/activitymonitor/SystemInfo.h | 2 +- src/bin/vmstat.cpp | 38 ++++++++++--------- .../virtualmemory/SettingsWindow.cpp | 9 +++-- src/system/kernel/system_info.cpp | 29 +------------- src/system/kernel/vm/VMAnonymousCache.cpp | 6 +-- src/system/kernel/vm/VMAnonymousCache.h | 2 +- src/system/kernel/vm/vm.cpp | 7 +--- src/system/kernel/vm/vm_page.cpp | 1 + src/system/libroot/os/system_info.cpp | 10 ----- 15 files changed, 55 insertions(+), 106 deletions(-) diff --git a/headers/os/kernel/OS.h b/headers/os/kernel/OS.h index d6199a3d6a..66131ee825 100644 --- a/headers/os/kernel/OS.h +++ b/headers/os/kernel/OS.h @@ -433,14 +433,15 @@ typedef struct { uint64 max_pages; /* total # of accessible pages */ uint64 used_pages; /* # of accessible pages in use */ - uint64 free_pages; - uint64 reserved_pages; uint64 cached_pages; uint64 block_cache_pages; uint64 ignored_pages; /* # of ignored/inaccessible pages */ + uint64 needed_memory; + uint64 free_memory; + uint64 max_swap_pages; - uint64 used_swap_pages; + uint64 free_swap_pages; uint32 page_faults; /* # of page faults */ diff --git a/headers/private/kernel/vm/vm.h b/headers/private/kernel/vm/vm.h index 1962f418a2..2b4b1e32c3 100644 --- a/headers/private/kernel/vm/vm.h +++ b/headers/private/kernel/vm/vm.h @@ -142,7 +142,7 @@ status_t vm_get_physical_page_debug(phys_addr_t paddr, addr_t* vaddr, void** _handle); status_t vm_put_physical_page_debug(addr_t vaddr, void* handle); -void vm_get_info(struct system_memory_info *info); +void vm_get_info(system_info *info); uint32 vm_num_page_faults(void); off_t vm_available_memory(void); off_t vm_available_not_needed_memory(void); diff --git a/headers/private/system/syscalls.h b/headers/private/system/syscalls.h index fd787c4cda..3f475e7886 100644 --- a/headers/private/system/syscalls.h +++ b/headers/private/system/syscalls.h @@ -521,8 +521,6 @@ extern status_t _kern_get_cpu_topology_info( cpu_topology_node_info* topologyInfos, uint32* topologyInfoCount); -extern status_t _kern_get_system_info_etc(int32 id, void *buffer, - size_t bufferSize); extern status_t _kern_analyze_scheduling(bigtime_t from, bigtime_t until, void* buffer, size_t size, struct scheduling_analysis* analysis); diff --git a/headers/private/system/system_info.h b/headers/private/system/system_info.h index 89a47f6bac..e04dd61f26 100644 --- a/headers/private/system/system_info.h +++ b/headers/private/system/system_info.h @@ -9,21 +9,6 @@ #include -#define B_MEMORY_INFO 'memo' - -struct system_memory_info { - uint64 max_memory; - uint64 free_memory; - uint64 needed_memory; - uint64 max_swap_space; - uint64 free_swap_space; - uint64 block_cache_memory; - uint32 page_faults; - - // TODO: add active/inactive page counts, swap in/out, ... -}; - - enum { // team creation or deletion; object == -1; either one also triggers on // exec() @@ -67,7 +52,6 @@ status_t __get_system_info(system_info* info); status_t __get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info); status_t __get_cpu_topology_info(cpu_topology_node_info* topologyInfos, uint32* topologyInfoCount); -status_t __get_system_info_etc(int32 id, void* buffer, size_t bufferSize); status_t __start_watching_system(int32 object, uint32 flags, port_id port, int32 token); diff --git a/src/apps/activitymonitor/DataSource.cpp b/src/apps/activitymonitor/DataSource.cpp index 9c954e1051..c493593f8c 100644 --- a/src/apps/activitymonitor/DataSource.cpp +++ b/src/apps/activitymonitor/DataSource.cpp @@ -497,13 +497,7 @@ BlockCacheDataSource::Copy() const int64 BlockCacheDataSource::NextValue(SystemInfo& info) { - system_memory_info memoryInfo; - status_t status = __get_system_info_etc(B_MEMORY_INFO, &memoryInfo, - sizeof(system_memory_info)); - if (status != B_OK) - return 0; - - return memoryInfo.block_cache_memory; + return info.BlockCacheMemory(); } diff --git a/src/apps/activitymonitor/SystemInfo.cpp b/src/apps/activitymonitor/SystemInfo.cpp index 2416e4a2a0..482fa694c1 100644 --- a/src/apps/activitymonitor/SystemInfo.cpp +++ b/src/apps/activitymonitor/SystemInfo.cpp @@ -26,8 +26,6 @@ SystemInfo::SystemInfo(SystemInfoHandler* handler) get_system_info(&fSystemInfo); fCPUInfos = new cpu_info[fSystemInfo.cpu_count]; get_cpu_info(0, fSystemInfo.cpu_count, fCPUInfos); - __get_system_info_etc(B_MEMORY_INFO, &fMemoryInfo, - sizeof(system_memory_info)); if (handler != NULL) { fRunningApps = handler->RunningApps(); @@ -50,45 +48,53 @@ uint64 SystemInfo::CachedMemory() const { #ifdef __HAIKU__ - return (uint64)fSystemInfo.cached_pages * B_PAGE_SIZE; + return fSystemInfo.cached_pages * B_PAGE_SIZE; #else return 0LL; #endif } +uint64 +SystemInfo::BlockCacheMemory() const +{ + return fSystemInfo.block_cache_pages * B_PAGE_SIZE; +} + + uint64 SystemInfo::UsedMemory() const { - return (uint64)fSystemInfo.used_pages * B_PAGE_SIZE; + return fSystemInfo.used_pages * B_PAGE_SIZE; } uint64 SystemInfo::MaxMemory() const { - return (uint64)fSystemInfo.max_pages * B_PAGE_SIZE; + return fSystemInfo.max_pages * B_PAGE_SIZE; } uint32 SystemInfo::PageFaults() const { - return fMemoryInfo.page_faults; + return fSystemInfo.page_faults; } uint64 SystemInfo::UsedSwapSpace() const { - return fMemoryInfo.max_swap_space - fMemoryInfo.free_swap_space; + return (fSystemInfo.max_swap_pages - fSystemInfo.free_swap_pages) + * B_PAGE_SIZE; } uint64 SystemInfo::MaxSwapSpace() const { - return fMemoryInfo.max_swap_space; + return fSystemInfo.max_swap_pages * B_PAGE_SIZE; } diff --git a/src/apps/activitymonitor/SystemInfo.h b/src/apps/activitymonitor/SystemInfo.h index 66e900b27b..065eb234bd 100644 --- a/src/apps/activitymonitor/SystemInfo.h +++ b/src/apps/activitymonitor/SystemInfo.h @@ -20,6 +20,7 @@ public: ~SystemInfo(); uint64 CachedMemory() const; + uint64 BlockCacheMemory() const; uint64 UsedMemory() const; uint64 MaxMemory() const; @@ -64,7 +65,6 @@ private: system_info fSystemInfo; cpu_info* fCPUInfos; - system_memory_info fMemoryInfo; bigtime_t fTime; bool fRetrievedNetwork; uint64 fBytesReceived; diff --git a/src/bin/vmstat.cpp b/src/bin/vmstat.cpp index ed2dee580f..9d6085b32f 100644 --- a/src/bin/vmstat.cpp +++ b/src/bin/vmstat.cpp @@ -66,40 +66,44 @@ main(int argc, char** argv) break; } } - system_memory_info info; - status_t status = __get_system_info_etc(B_MEMORY_INFO, &info, - sizeof(system_memory_info)); + system_info info; + status_t status = get_system_info(&info); if (status != B_OK) { fprintf(stderr, "%s: cannot get system info: %s\n", kProgramName, strerror(status)); return 1; } - printf("max memory:\t\t%Lu\n", info.max_memory); + printf("max memory:\t\t%Lu\n", info.max_pages * B_PAGE_SIZE); printf("free memory:\t\t%Lu\n", info.free_memory); printf("needed memory:\t\t%Lu\n", info.needed_memory); - printf("block cache memory:\t%Lu\n", info.block_cache_memory); - printf("max swap space:\t\t%Lu\n", info.max_swap_space); - printf("free swap space:\t%Lu\n", info.free_swap_space); + printf("block cache memory:\t%Lu\n", info.block_cache_pages * B_PAGE_SIZE); + printf("max swap space:\t\t%Lu\n", info.max_swap_pages * B_PAGE_SIZE); + printf("free swap space:\t%Lu\n", info.free_swap_pages * B_PAGE_SIZE); printf("page faults:\t\t%lu\n", info.page_faults); if (periodically) { puts("\npage faults used memory used swap block cache"); - system_memory_info lastInfo = info; + system_info lastInfo = info; while (true) { snooze(rate); - __get_system_info_etc(B_MEMORY_INFO, &info, - sizeof(system_memory_info)); + get_system_info(&info); - printf("%11ld %11Ld %11Ld %11Ld\n", - (int32)info.page_faults - lastInfo.page_faults, - (info.max_memory - info.free_memory) - - (lastInfo.max_memory - lastInfo.free_memory), - (info.max_swap_space - info.free_swap_space) - - (lastInfo.max_swap_space - lastInfo.free_swap_space), - info.block_cache_memory - lastInfo.block_cache_memory); + int32 pageFaults = info.page_faults - lastInfo.page_faults; + int64 usedMemory + = (info.max_pages * B_PAGE_SIZE - info.free_memory) + - (lastInfo.max_pages * B_PAGE_SIZE - lastInfo.free_memory); + int64 usedSwap + = ((info.max_swap_pages - info.free_swap_pages) + - (lastInfo.max_swap_pages - lastInfo.free_swap_pages)) + * B_PAGE_SIZE; + int64 blockCache + = (info.block_cache_pages - lastInfo.block_cache_pages) + * B_PAGE_SIZE; + printf("%11" B_PRId32 " %11" B_PRId64 " %11" B_PRId64 " %11" + B_PRId64 "\n", pageFaults, usedMemory, usedSwap, blockCache); lastInfo = info; } diff --git a/src/preferences/virtualmemory/SettingsWindow.cpp b/src/preferences/virtualmemory/SettingsWindow.cpp index 38bf127c48..508cf46060 100644 --- a/src/preferences/virtualmemory/SettingsWindow.cpp +++ b/src/preferences/virtualmemory/SettingsWindow.cpp @@ -476,11 +476,12 @@ SettingsWindow::_Update() void SettingsWindow::_UpdateSwapInfo() { - system_memory_info memInfo = {}; - __get_system_info_etc(B_MEMORY_INFO, &memInfo, sizeof(memInfo)); + system_info info; + get_system_info(&info); - off_t currentSwapSize = memInfo.max_swap_space; - off_t currentSwapUsed = (memInfo.max_swap_space - memInfo.free_swap_space); + off_t currentSwapSize = info.max_swap_pages * B_PAGE_SIZE; + off_t currentSwapUsed + = (info.max_swap_pages - info.free_swap_pages) * B_PAGE_SIZE; char sizeStr[16]; BString swapSizeStr = string_for_size(currentSwapSize, sizeStr, diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index 709fe30860..82929d0c4b 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -418,6 +418,7 @@ get_system_info(system_info* info) info->cpu_count = smp_get_num_cpus(); vm_page_get_stats(info); + vm_get_info(info); info->used_threads = thread_used_threads(); info->max_threads = thread_max_threads(); @@ -428,8 +429,6 @@ get_system_info(system_info* info) info->used_sems = sem_used_sems(); info->max_sems = sem_max_sems(); - // TODO: fill the new fields - info->kernel_version = kKernelVersion; strlcpy(info->kernel_name, kKernelName, B_FILE_NAME_LENGTH); strlcpy(info->kernel_build_date, __DATE__, B_OS_NAME_LENGTH); @@ -625,32 +624,6 @@ _user_get_cpu_topology_info(cpu_topology_node_info* topologyInfos, } -status_t -_user_get_system_info_etc(int32 id, void* userInfo, size_t size) -{ - if (userInfo == NULL || !IS_USER_ADDRESS(userInfo)) - return B_BAD_ADDRESS; - - switch (id) { - case B_MEMORY_INFO: - { - if (size < sizeof(system_memory_info)) - return B_BAD_VALUE; - - system_memory_info info; - vm_get_info(&info); - - info.block_cache_memory = block_cache_used_memory(); - - return user_memcpy(userInfo, &info, sizeof(system_memory_info)); - } - - default: - return B_BAD_VALUE; - } -} - - status_t _user_start_watching_system(int32 object, uint32 flags, port_id port, int32 token) diff --git a/src/system/kernel/vm/VMAnonymousCache.cpp b/src/system/kernel/vm/VMAnonymousCache.cpp index d0337ba71c..0abb92a8db 100644 --- a/src/system/kernel/vm/VMAnonymousCache.cpp +++ b/src/system/kernel/vm/VMAnonymousCache.cpp @@ -1667,11 +1667,11 @@ swap_total_swap_pages() void -swap_get_info(struct system_memory_info* info) +swap_get_info(system_info* info) { #if ENABLE_SWAP_SUPPORT - info->max_swap_space = (uint64)swap_total_swap_pages() * B_PAGE_SIZE; - info->free_swap_space = (uint64)swap_available_pages() * B_PAGE_SIZE; + info->max_swap_pages = swap_total_swap_pages(); + info->free_swap_pages = swap_available_pages(); #else info->max_swap_space = 0; info->free_swap_space = 0; diff --git a/src/system/kernel/vm/VMAnonymousCache.h b/src/system/kernel/vm/VMAnonymousCache.h index 065f422325..b68800047c 100644 --- a/src/system/kernel/vm/VMAnonymousCache.h +++ b/src/system/kernel/vm/VMAnonymousCache.h @@ -99,7 +99,7 @@ private: #endif // ENABLE_SWAP_SUPPORT -extern "C" void swap_get_info(struct system_memory_info* info); +extern "C" void swap_get_info(system_info* info); #endif /* _KERNEL_VM_STORE_ANONYMOUS_H */ diff --git a/src/system/kernel/vm/vm.cpp b/src/system/kernel/vm/vm.cpp index 2e5a3747dd..5de32b25e8 100644 --- a/src/system/kernel/vm/vm.cpp +++ b/src/system/kernel/vm/vm.cpp @@ -4628,16 +4628,13 @@ vm_put_physical_page_debug(addr_t vaddr, void* handle) void -vm_get_info(system_memory_info* info) +vm_get_info(system_info* info) { swap_get_info(info); - info->max_memory = vm_page_num_pages() * B_PAGE_SIZE; - info->page_faults = sPageFaults; - MutexLocker locker(sAvailableMemoryLock); - info->free_memory = sAvailableMemory; info->needed_memory = sNeededMemory; + info->free_memory = sAvailableMemory; } diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 9f1f548887..008782773c 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -4059,6 +4059,7 @@ vm_page_get_stats(system_info *info) // TODO: We should subtract the blocks that are in use ATM, since those // can't really be freed in a low memory situation. page_num_t blockCachePages = block_cache_used_memory() / B_PAGE_SIZE; + info->block_cache_pages = blockCachePages; // Non-temporary modified pages are special as they represent pages that // can be written back, so they could be freed if necessary, for us diff --git a/src/system/libroot/os/system_info.cpp b/src/system/libroot/os/system_info.cpp index adf7a0c710..337e7469a9 100644 --- a/src/system/libroot/os/system_info.cpp +++ b/src/system/libroot/os/system_info.cpp @@ -198,16 +198,6 @@ __get_cpu_topology_info(cpu_topology_node_info* topologyInfos, } -status_t -__get_system_info_etc(int32 id, void *info, size_t size) -{ - if (info == NULL || size == 0 || id < 0) - return B_BAD_VALUE; - - return _kern_get_system_info_etc(id, info, size); -} - - status_t __start_watching_system(int32 object, uint32 flags, port_id port, int32 token) { From 7b9668170c140267861978da69ac0bb6f07f1373 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 16 Dec 2013 05:00:02 +0100 Subject: [PATCH 198/273] kernel, boot: Increase CPU limit to 64 --- .../kernel/boot/platform/bios_ia32/platform_kernel_args.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h b/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h index 6772222f01..31861be537 100644 --- a/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h +++ b/headers/private/kernel/boot/platform/bios_ia32/platform_kernel_args.h @@ -15,7 +15,7 @@ #include -#define SMP_MAX_CPUS 8 +#define SMP_MAX_CPUS 64 #define MAX_PHYSICAL_MEMORY_RANGE 32 #define MAX_PHYSICAL_ALLOCATED_RANGE 32 From 611376fef7e00967fb65342802ba668a807348d5 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 17 Dec 2013 02:55:22 +0100 Subject: [PATCH 199/273] x86: Let each CPU have its own GDT --- .../private/kernel/arch/x86/32/descriptors.h | 50 +- headers/private/kernel/arch/x86/32/iframe.h | 2 +- .../private/kernel/arch/x86/64/descriptors.h | 25 +- headers/private/kernel/arch/x86/arch_cpu.h | 1 + headers/private/kernel/arch/x86/descriptors.h | 10 +- .../boot/platform/bios_ia32/interrupts.cpp | 3 +- src/system/boot/platform/bios_ia32/long.cpp | 8 +- src/system/boot/platform/bios_ia32/long_asm.S | 4 +- src/system/boot/platform/bios_ia32/mmu.cpp | 16 +- src/system/kernel/arch/x86/32/apm.cpp | 68 +- src/system/kernel/arch/x86/32/descriptors.cpp | 736 +++++++++--------- src/system/kernel/arch/x86/32/interrupts.S | 14 +- src/system/kernel/arch/x86/32/syscalls.cpp | 2 +- src/system/kernel/arch/x86/32/thread.cpp | 21 +- src/system/kernel/arch/x86/64/descriptors.cpp | 8 +- src/system/kernel/arch/x86/64/interrupts.S | 4 +- src/system/kernel/arch/x86/64/syscalls.cpp | 12 +- src/system/kernel/arch/x86/64/thread.cpp | 4 +- src/system/kernel/arch/x86/arch_cpu.cpp | 3 +- 19 files changed, 522 insertions(+), 469 deletions(-) diff --git a/headers/private/kernel/arch/x86/32/descriptors.h b/headers/private/kernel/arch/x86/32/descriptors.h index e09b849e3a..3bdc8c3310 100644 --- a/headers/private/kernel/arch/x86/32/descriptors.h +++ b/headers/private/kernel/arch/x86/32/descriptors.h @@ -9,30 +9,40 @@ #define _KERNEL_ARCH_X86_32_DESCRIPTORS_H -#define KERNEL_CODE_SEG 0x8 -#define KERNEL_DATA_SEG 0x10 +// Segments common for all CPUs. +#define KERNEL_CODE_SEGMENT 1 +#define KERNEL_DATA_SEGMENT 2 -#define USER_CODE_SEG 0x1b -#define USER_DATA_SEG 0x23 +#define USER_CODE_SEGMENT 3 +#define USER_DATA_SEGMENT 4 -#define APM_CODE32_SEGMENT 0x28 -#define APM_CODE16_SEGMENT 0x30 -#define APM_DATA_SEGMENT 0x38 +#define APM_CODE32_SEGMENT 5 +#define APM_CODE16_SEGMENT 6 +#define APM_DATA_SEGMENT 7 -#define BIOS_DATA_SEGMENT 0x40 +#define BIOS_DATA_SEGMENT 8 + +// Per-CPU segments. +#define TSS_SEGMENT 9 +#define DOUBLE_FAULT_TSS_SEGMENT 10 +#define KERNEL_TLS_SEGMENT 11 +#define USER_TLS_SEGMENT 12 +#define APM_SEGMENT 13 + +#define GDT_SEGMENT_COUNT 14 + + +#define KERNEL_CODE_SELECTOR ((KERNEL_CODE_SEGMENT << 3) | DPL_KERNEL) +#define KERNEL_DATA_SELECTOR ((KERNEL_DATA_SEGMENT << 3) | DPL_KERNEL) + +#define USER_CODE_SELECTOR ((USER_CODE_SEGMENT << 3) | DPL_USER) +#define USER_DATA_SELECTOR ((USER_DATA_SEGMENT << 3) | DPL_USER) #ifndef _ASSEMBLER // this file can also be included from assembler as well // (and is in arch_interrupts.S) -#define DOUBLE_FAULT_TSS_BASE_SEGMENT 9 -#define TSS_BASE_SEGMENT (DOUBLE_FAULT_TSS_BASE_SEGMENT + smp_get_num_cpus()) -#define TLS_BASE_SEGMENT (TSS_BASE_SEGMENT + smp_get_num_cpus()) -#define APM_BASE_SEGMENT (TLS_BASE_SEGMENT + smp_get_num_cpus()) - -#define TSS_SEGMENT(cpu) (TSS_BASE_SEGMENT + cpu) - // defines entries in the GDT/LDT struct segment_descriptor { @@ -73,6 +83,9 @@ struct tss { uint16 io_map_base; }; +typedef segment_descriptor global_descriptor_table[GDT_SEGMENT_COUNT]; +extern global_descriptor_table gGDTs[]; + static inline void clear_segment_descriptor(segment_descriptor* desc) @@ -141,6 +154,13 @@ set_tss_descriptor(segment_descriptor* desc, addr_t base, uint32 limit) } +static inline segment_descriptor* +get_gdt(int32 cpu) +{ + return gGDTs[cpu]; +} + + #endif /* _ASSEMBLER */ #endif /* _KERNEL_ARCH_X86_32_DESCRIPTORS_H */ diff --git a/headers/private/kernel/arch/x86/32/iframe.h b/headers/private/kernel/arch/x86/32/iframe.h index 47c423826f..8362b75d83 100644 --- a/headers/private/kernel/arch/x86/32/iframe.h +++ b/headers/private/kernel/arch/x86/32/iframe.h @@ -37,7 +37,7 @@ struct iframe { uint32 user_ss; }; -#define IFRAME_IS_USER(f) ((f)->cs == USER_CODE_SEG \ +#define IFRAME_IS_USER(f) ((f)->cs == USER_CODE_SELECTOR \ || ((f)->flags & 0x20000) != 0) diff --git a/headers/private/kernel/arch/x86/64/descriptors.h b/headers/private/kernel/arch/x86/64/descriptors.h index 3f05196586..fc9e5d0177 100644 --- a/headers/private/kernel/arch/x86/64/descriptors.h +++ b/headers/private/kernel/arch/x86/64/descriptors.h @@ -8,19 +8,28 @@ // Segment definitions. // Note that the ordering of these is important to SYSCALL/SYSRET. -#define KERNEL_CODE_SEG 0x08 -#define KERNEL_DATA_SEG 0x10 -#define USER_DATA_SEG 0x1b -#define USER_CODE_SEG 0x23 +#define KERNEL_CODE_SEGMENT 1 +#define KERNEL_DATA_SEGMENT 2 +#define USER_DATA_SEGMENT 3 +#define USER_CODE_SEGMENT 4 + +#define TSS_BASE_SEGMENT 5 + +#define TSS_SEGMENT(cpu) (TSS_BASE_SEGMENT + cpu * 2) + +#define GDT_SEGMENT_COUNT (TSS_BASE_SEGMENT + SMP_MAX_CPUS * 2) + + +#define KERNEL_CODE_SELECTOR ((KERNEL_CODE_SEGMENT << 3) | DPL_KERNEL) +#define KERNEL_DATA_SELECTOR ((KERNEL_DATA_SEGMENT << 3) | DPL_KERNEL) + +#define USER_CODE_SELECTOR ((USER_CODE_SEGMENT << 3) | DPL_USER) +#define USER_DATA_SELECTOR ((USER_DATA_SEGMENT << 3) | DPL_USER) #ifndef _ASSEMBLER -#define TSS_BASE_SEGMENT 5 -#define TSS_SEGMENT(cpu) (TSS_BASE_SEGMENT + cpu * 2) - - // Structure of a segment descriptor. struct segment_descriptor { uint32 limit0 : 16; diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 502b33b6c2..516b5fc52d 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -335,6 +335,7 @@ typedef struct arch_cpu_info { struct tss tss; #ifndef __x86_64__ struct tss double_fault_tss; + void* kernel_tls; #endif } arch_cpu_info; diff --git a/headers/private/kernel/arch/x86/descriptors.h b/headers/private/kernel/arch/x86/descriptors.h index 30dc9f657f..5b16ab7754 100644 --- a/headers/private/kernel/arch/x86/descriptors.h +++ b/headers/private/kernel/arch/x86/descriptors.h @@ -9,6 +9,10 @@ #define _KERNEL_ARCH_x86_DESCRIPTORS_H +#define DPL_KERNEL 0 +#define DPL_USER 3 + + #ifndef _ASSEMBLER @@ -18,11 +22,6 @@ struct kernel_args; -enum descriptor_privilege_levels { - DPL_KERNEL = 0, - DPL_USER = 3, -}; - enum descriptor_types { // segment types DT_CODE_EXECUTE_ONLY = 0x8, @@ -48,6 +47,7 @@ enum gate_types { }; +void x86_descriptors_preboot_init_percpu(kernel_args* args, int cpu); void x86_descriptors_init(kernel_args* args); void x86_descriptors_init_percpu(kernel_args* args, int cpu); status_t x86_descriptors_init_post_vm(kernel_args* args); diff --git a/src/system/boot/platform/bios_ia32/interrupts.cpp b/src/system/boot/platform/bios_ia32/interrupts.cpp index 2d5d0fa596..92f6d6f7f9 100644 --- a/src/system/boot/platform/bios_ia32/interrupts.cpp +++ b/src/system/boot/platform/bios_ia32/interrupts.cpp @@ -96,7 +96,8 @@ set_interrupt_gate(int n, void (*function)()) if (n >= DEBUG_IDT_SLOT_COUNT) return; - sDebugIDT[n].a = (KERNEL_CODE_SEG << 16) | (0x0000ffff & (addr_t)function); + sDebugIDT[n].a + = (KERNEL_CODE_SELECTOR << 16) | (0x0000ffff & (addr_t)function); sDebugIDT[n].b = (0xffff0000 & (addr_t)function) | 0x8e00; } diff --git a/src/system/boot/platform/bios_ia32/long.cpp b/src/system/boot/platform/bios_ia32/long.cpp index 3db5220e36..990bcdd546 100644 --- a/src/system/boot/platform/bios_ia32/long.cpp +++ b/src/system/boot/platform/bios_ia32/long.cpp @@ -74,13 +74,13 @@ long_gdt_init() clear_segment_descriptor(&gdt[0]); // Set up code/data segments (TSS segments set up later in the kernel). - set_segment_descriptor(&gdt[KERNEL_CODE_SEG / 8], DT_CODE_EXECUTE_ONLY, + set_segment_descriptor(&gdt[KERNEL_CODE_SEGMENT], DT_CODE_EXECUTE_ONLY, DPL_KERNEL); - set_segment_descriptor(&gdt[KERNEL_DATA_SEG / 8], DT_DATA_WRITEABLE, + set_segment_descriptor(&gdt[KERNEL_DATA_SEGMENT], DT_DATA_WRITEABLE, DPL_KERNEL); - set_segment_descriptor(&gdt[USER_CODE_SEG / 8], DT_CODE_EXECUTE_ONLY, + set_segment_descriptor(&gdt[USER_CODE_SEGMENT], DT_CODE_EXECUTE_ONLY, DPL_USER); - set_segment_descriptor(&gdt[USER_DATA_SEG / 8], DT_DATA_WRITEABLE, + set_segment_descriptor(&gdt[USER_DATA_SEGMENT], DT_DATA_WRITEABLE, DPL_USER); // Used by long_enter_kernel(). diff --git a/src/system/boot/platform/bios_ia32/long_asm.S b/src/system/boot/platform/bios_ia32/long_asm.S index 72b30c26cf..0ee9b8e9be 100644 --- a/src/system/boot/platform/bios_ia32/long_asm.S +++ b/src/system/boot/platform/bios_ia32/long_asm.S @@ -58,12 +58,12 @@ FUNCTION(long_enter_kernel): movl %ecx, %cr0 // Jump into the 64-bit code segment. - ljmp $KERNEL_CODE_SEG, $.Llmode + ljmp $KERNEL_CODE_SELECTOR, $.Llmode .align 8 .code64 .Llmode: // Set data segments. - mov $KERNEL_DATA_SEG, %ax + mov $KERNEL_DATA_SELECTOR, %ax mov %ax, %ss xor %ax, %ax mov %ax, %ds diff --git a/src/system/boot/platform/bios_ia32/mmu.cpp b/src/system/boot/platform/bios_ia32/mmu.cpp index 93eaed5423..8bfe21ec09 100644 --- a/src/system/boot/platform/bios_ia32/mmu.cpp +++ b/src/system/boot/platform/bios_ia32/mmu.cpp @@ -577,20 +577,20 @@ mmu_init_for_kernel(void) clear_segment_descriptor(&virtualGDT[0]); // seg 0x08 - kernel 4GB code - set_segment_descriptor(&virtualGDT[1], 0, 0xffffffff, DT_CODE_READABLE, - DPL_KERNEL); + set_segment_descriptor(&virtualGDT[KERNEL_CODE_SEGMENT], 0, 0xffffffff, + DT_CODE_READABLE, DPL_KERNEL); // seg 0x10 - kernel 4GB data - set_segment_descriptor(&virtualGDT[2], 0, 0xffffffff, DT_DATA_WRITEABLE, - DPL_KERNEL); + set_segment_descriptor(&virtualGDT[KERNEL_DATA_SEGMENT], 0, 0xffffffff, + DT_DATA_WRITEABLE, DPL_KERNEL); // seg 0x1b - ring 3 user 4GB code - set_segment_descriptor(&virtualGDT[3], 0, 0xffffffff, DT_CODE_READABLE, - DPL_USER); + set_segment_descriptor(&virtualGDT[USER_CODE_SEGMENT], 0, 0xffffffff, + DT_CODE_READABLE, DPL_USER); // seg 0x23 - ring 3 user 4GB data - set_segment_descriptor(&virtualGDT[4], 0, 0xffffffff, DT_DATA_WRITEABLE, - DPL_USER); + set_segment_descriptor(&virtualGDT[USER_DATA_SEGMENT], 0, 0xffffffff, + DT_DATA_WRITEABLE, DPL_USER); // virtualGDT[5] and above will be filled later by the kernel // to contain the TSS descriptors, and for TLS (one for every CPU) diff --git a/src/system/kernel/arch/x86/32/apm.cpp b/src/system/kernel/arch/x86/32/apm.cpp index 1c6b72b178..c5174485da 100644 --- a/src/system/kernel/arch/x86/32/apm.cpp +++ b/src/system/kernel/arch/x86/32/apm.cpp @@ -33,7 +33,6 @@ #define CARRY_FLAG 0x01 -extern segment_descriptor *gGDT; extern void *gDmaAddress; extern addr_t gBiosBase; @@ -314,49 +313,52 @@ apm_init(kernel_args *args) // (the first 640kB are mapped as DMA area in arch_vm_init()). addr_t biosData = (addr_t)gDmaAddress + 0x400; - set_segment_descriptor(&gGDT[BIOS_DATA_SEGMENT >> 3], - biosData, B_PAGE_SIZE - biosData, - DT_DATA_WRITEABLE, DPL_KERNEL); + for (uint32 i = 0; i < args->num_cpus; i++) { + segment_descriptor* gdt = get_gdt(i); - // TODO: test if APM segments really are in the BIOS ROM area (especially the - // data segment) + set_segment_descriptor(&gdt[BIOS_DATA_SEGMENT], biosData, + B_PAGE_SIZE - biosData, DT_DATA_WRITEABLE, DPL_KERNEL); - // Setup APM GDTs + // TODO: test if APM segments really are in the BIOS ROM area + // (especially the data segment) - // We ignore their length, and just set their segments to 64 kB which - // shouldn't cause any headaches + // Setup APM GDTs - set_segment_descriptor(&gGDT[APM_CODE32_SEGMENT >> 3], - gBiosBase + (info.code32_segment_base << 4) - 0xe0000, 0xffff, - DT_CODE_READABLE, DPL_KERNEL); - set_segment_descriptor(&gGDT[APM_CODE16_SEGMENT >> 3], - gBiosBase + (info.code16_segment_base << 4) - 0xe0000, 0xffff, - DT_CODE_READABLE, DPL_KERNEL); - gGDT[APM_CODE16_SEGMENT >> 3].d_b = 0; - // 16-bit segment + // We ignore their length, and just set their segments to 64 kB which + // shouldn't cause any headaches - if ((info.data_segment_base << 4) < 0xe0000) { - // use the BIOS data segment as data segment for APM + set_segment_descriptor(&gdt[APM_CODE32_SEGMENT], + gBiosBase + (info.code32_segment_base << 4) - 0xe0000, 0xffff, + DT_CODE_READABLE, DPL_KERNEL); + set_segment_descriptor(&gdt[APM_CODE16_SEGMENT], + gBiosBase + (info.code16_segment_base << 4) - 0xe0000, 0xffff, + DT_CODE_READABLE, DPL_KERNEL); + gdt[APM_CODE16_SEGMENT].d_b = 0; + // 16-bit segment - if (info.data_segment_length == 0) { - args->platform_args.apm.data_segment_length = B_PAGE_SIZE - - info.data_segment_base; + if ((info.data_segment_base << 4) < 0xe0000) { + // use the BIOS data segment as data segment for APM + + if (info.data_segment_length == 0) { + args->platform_args.apm.data_segment_length = B_PAGE_SIZE + - info.data_segment_base; + } + + set_segment_descriptor(&gdt[APM_DATA_SEGMENT], + (addr_t)gDmaAddress + (info.data_segment_base << 4), + info.data_segment_length, + DT_DATA_WRITEABLE, DPL_KERNEL); + } else { + // use the BIOS area as data segment + set_segment_descriptor(&gdt[APM_DATA_SEGMENT], + gBiosBase + (info.data_segment_base << 4) - 0xe0000, 0xffff, + DT_DATA_WRITEABLE, DPL_KERNEL); } - - set_segment_descriptor(&gGDT[APM_DATA_SEGMENT >> 3], - (addr_t)gDmaAddress + (info.data_segment_base << 4), - info.data_segment_length, - DT_DATA_WRITEABLE, DPL_KERNEL); - } else { - // use the BIOS area as data segment - set_segment_descriptor(&gGDT[APM_DATA_SEGMENT >> 3], - gBiosBase + (info.data_segment_base << 4) - 0xe0000, 0xffff, - DT_DATA_WRITEABLE, DPL_KERNEL); } // setup APM entry point - sAPMBiosEntry.segment = APM_CODE32_SEGMENT; + sAPMBiosEntry.segment = (APM_CODE32_SEGMENT << 3) | DPL_KERNEL; sAPMBiosEntry.offset = info.code32_segment_offset; apm_driver_version(info.version); diff --git a/src/system/kernel/arch/x86/32/descriptors.cpp b/src/system/kernel/arch/x86/32/descriptors.cpp index 53b60e4df8..fef30fa3d4 100644 --- a/src/system/kernel/arch/x86/32/descriptors.cpp +++ b/src/system/kernel/arch/x86/32/descriptors.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2008-2011, Michael Lotz, mmlr@mlotz.ch. * Copyright 2010, Clemens Zeidler, haiku@clemens-zeidler.de. * Copyright 2009-2011, Ingo Weinhold, ingo_weinhold@gmx.de. @@ -28,16 +29,20 @@ #include "interrupts.h" -static interrupt_descriptor* sIDTs[SMP_MAX_CPUS]; +#define IDT_GATES_COUNT 256 + + +typedef interrupt_descriptor interrupt_descriptor_table[IDT_GATES_COUNT]; + +global_descriptor_table gGDTs[SMP_MAX_CPUS]; +static interrupt_descriptor_table sIDTs[SMP_MAX_CPUS]; // table with functions handling respective interrupts typedef void interrupt_handler_function(struct iframe* frame); -static const uint32 kInterruptHandlerTableSize = 256; +static const uint32 kInterruptHandlerTableSize = IDT_GATES_COUNT; interrupt_handler_function* gInterruptHandlerTable[kInterruptHandlerTableSize]; -segment_descriptor* gGDT; - /*! Initializes a descriptor in an IDT. */ @@ -47,7 +52,7 @@ set_gate(interrupt_descriptor *gate_addr, addr_t addr, int type, int dpl) unsigned int gate1; // first byte of gate desc unsigned int gate2; // second byte of gate desc - gate1 = (KERNEL_CODE_SEG << 16) | (0x0000ffff & addr); + gate1 = (KERNEL_CODE_SELECTOR << 16) | (0x0000ffff & addr); gate2 = (0xffff0000 & addr) | 0x8000 | (dpl << 13) | (type << 8); gate_addr->a = gate1; @@ -95,9 +100,9 @@ set_task_gate(int32 cpu, int32 n, int32 segment) static void -load_tss(int cpu) +load_tss() { - short seg = (TSS_SEGMENT(cpu) << 3) | DPL_KERNEL; + short seg = (TSS_SEGMENT << 3) | DPL_KERNEL; asm("ltr %%ax" : : "a" (seg)); } @@ -179,304 +184,389 @@ init_double_fault(int cpuNum) size_t stackSize; tss->sp0 = (addr_t)x86_get_double_fault_stack(cpuNum, &stackSize); tss->sp0 += stackSize; - tss->ss0 = KERNEL_DATA_SEG; + tss->ss0 = KERNEL_DATA_SELECTOR; tss->cr3 = x86_read_cr3(); // copy the current cr3 to the double fault cr3 tss->eip = (uint32)&double_fault; - tss->es = KERNEL_DATA_SEG; - tss->cs = KERNEL_CODE_SEG; - tss->ss = KERNEL_DATA_SEG; + tss->es = KERNEL_DATA_SELECTOR; + tss->cs = KERNEL_CODE_SELECTOR; + tss->ss = KERNEL_DATA_SELECTOR; tss->esp = tss->sp0; - tss->ds = KERNEL_DATA_SEG; - tss->fs = KERNEL_DATA_SEG; - tss->gs = KERNEL_DATA_SEG; + tss->ds = KERNEL_DATA_SELECTOR; + tss->fs = KERNEL_DATA_SELECTOR; + tss->gs = KERNEL_DATA_SELECTOR; tss->ldt_seg_selector = 0; tss->io_map_base = sizeof(struct tss); // add TSS descriptor for this new TSS - uint16 tssSegmentDescriptorIndex = DOUBLE_FAULT_TSS_BASE_SEGMENT + cpuNum; - set_tss_descriptor(&gGDT[tssSegmentDescriptorIndex], - (addr_t)tss, sizeof(struct tss)); + set_tss_descriptor(&gGDTs[cpuNum][DOUBLE_FAULT_TSS_SEGMENT], (addr_t)tss, + sizeof(struct tss)); - set_task_gate(cpuNum, 8, tssSegmentDescriptorIndex << 3); + set_task_gate(cpuNum, 8, DOUBLE_FAULT_TSS_SEGMENT << 3); +} + + +static void +load_gdt(int cpu) +{ + struct { + uint16 limit; + void* address; + } _PACKED gdt_descriptor = { + GDT_SEGMENT_COUNT * sizeof(segment_descriptor) - 1, + gGDTs[cpu] + }; + + asm volatile("lgdt %0" : : "m" (gdt_descriptor)); +} + + +static void +init_gdt_percpu(kernel_args* args, int cpu) +{ + STATIC_ASSERT(GDT_SEGMENT_COUNT <= 8192); + + segment_descriptor* gdt = get_gdt(cpu); + + clear_segment_descriptor(&gdt[0]); + + set_segment_descriptor(&gdt[KERNEL_CODE_SEGMENT], 0, addr_t(-1), + DT_CODE_READABLE, DPL_KERNEL); + set_segment_descriptor(&gdt[KERNEL_DATA_SEGMENT], 0, addr_t(-1), + DT_DATA_WRITEABLE, DPL_KERNEL); + + set_segment_descriptor(&gdt[USER_CODE_SEGMENT], 0, addr_t(-1), + DT_CODE_READABLE, DPL_USER); + set_segment_descriptor(&gdt[USER_DATA_SEGMENT], 0, addr_t(-1), + DT_DATA_WRITEABLE, DPL_USER); + + // initialize the regular and double fault tss stored in the per-cpu + // structure + memset(&gCPU[cpu].arch.tss, 0, sizeof(struct tss)); + gCPU[cpu].arch.tss.ss0 = (KERNEL_DATA_SEGMENT << 3) | DPL_KERNEL; + gCPU[cpu].arch.tss.io_map_base = sizeof(struct tss); + + // add TSS descriptor for this new TSS + set_tss_descriptor(&gdt[TSS_SEGMENT], (addr_t)&gCPU[cpu].arch.tss, + sizeof(struct tss)); + + // initialize the double fault tss + init_double_fault(cpu); + + set_segment_descriptor(&gdt[KERNEL_TLS_SEGMENT], + (addr_t)&gCPU[cpu].arch.kernel_tls, sizeof(void*), DT_DATA_WRITEABLE, + DPL_KERNEL); + set_segment_descriptor(&gdt[USER_TLS_SEGMENT], 0, TLS_SIZE, + DT_DATA_WRITEABLE, DPL_USER); + + load_gdt(cpu); + + load_tss(); + + // set kernel TLS segment + asm volatile("movw %0, %%gs" : : "r" (KERNEL_TLS_SEGMENT << 3)); +} + + +static void +load_idt(int cpu) +{ + struct { + uint16 limit; + void* address; + } _PACKED idt_descriptor = { + IDT_GATES_COUNT * sizeof(interrupt_descriptor) - 1, + &sIDTs[cpu] + }; + + asm volatile("lidt %0" : : "m" (idt_descriptor)); +} + + +static void +init_idt_percpu(kernel_args* args, int cpu) +{ + set_interrupt_gate(cpu, 0, &trap0); + set_interrupt_gate(cpu, 1, &trap1); + set_interrupt_gate(cpu, 2, &trap2); + set_trap_gate(cpu, 3, &trap3); + set_interrupt_gate(cpu, 4, &trap4); + set_interrupt_gate(cpu, 5, &trap5); + set_interrupt_gate(cpu, 6, &trap6); + set_interrupt_gate(cpu, 7, &trap7); + // trap8 (double fault) is set in init_double_fault(). + set_interrupt_gate(cpu, 9, &trap9); + set_interrupt_gate(cpu, 10, &trap10); + set_interrupt_gate(cpu, 11, &trap11); + set_interrupt_gate(cpu, 12, &trap12); + set_interrupt_gate(cpu, 13, &trap13); + set_interrupt_gate(cpu, 14, &trap14); + //set_interrupt_gate(cpu, 15, &trap15); + set_interrupt_gate(cpu, 16, &trap16); + set_interrupt_gate(cpu, 17, &trap17); + set_interrupt_gate(cpu, 18, &trap18); + set_interrupt_gate(cpu, 19, &trap19); + + // legacy or ioapic interrupts + set_interrupt_gate(cpu, 32, &trap32); + set_interrupt_gate(cpu, 33, &trap33); + set_interrupt_gate(cpu, 34, &trap34); + set_interrupt_gate(cpu, 35, &trap35); + set_interrupt_gate(cpu, 36, &trap36); + set_interrupt_gate(cpu, 37, &trap37); + set_interrupt_gate(cpu, 38, &trap38); + set_interrupt_gate(cpu, 39, &trap39); + set_interrupt_gate(cpu, 40, &trap40); + set_interrupt_gate(cpu, 41, &trap41); + set_interrupt_gate(cpu, 42, &trap42); + set_interrupt_gate(cpu, 43, &trap43); + set_interrupt_gate(cpu, 44, &trap44); + set_interrupt_gate(cpu, 45, &trap45); + set_interrupt_gate(cpu, 46, &trap46); + set_interrupt_gate(cpu, 47, &trap47); + + // additional ioapic interrupts + set_interrupt_gate(cpu, 48, &trap48); + set_interrupt_gate(cpu, 49, &trap49); + set_interrupt_gate(cpu, 50, &trap50); + set_interrupt_gate(cpu, 51, &trap51); + set_interrupt_gate(cpu, 52, &trap52); + set_interrupt_gate(cpu, 53, &trap53); + set_interrupt_gate(cpu, 54, &trap54); + set_interrupt_gate(cpu, 55, &trap55); + + // configurable msi or msi-x interrupts + set_interrupt_gate(cpu, 56, &trap56); + set_interrupt_gate(cpu, 57, &trap57); + set_interrupt_gate(cpu, 58, &trap58); + set_interrupt_gate(cpu, 59, &trap59); + set_interrupt_gate(cpu, 60, &trap60); + set_interrupt_gate(cpu, 61, &trap61); + set_interrupt_gate(cpu, 62, &trap62); + set_interrupt_gate(cpu, 63, &trap63); + set_interrupt_gate(cpu, 64, &trap64); + set_interrupt_gate(cpu, 65, &trap65); + set_interrupt_gate(cpu, 66, &trap66); + set_interrupt_gate(cpu, 67, &trap67); + set_interrupt_gate(cpu, 68, &trap68); + set_interrupt_gate(cpu, 69, &trap69); + set_interrupt_gate(cpu, 70, &trap70); + set_interrupt_gate(cpu, 71, &trap71); + set_interrupt_gate(cpu, 72, &trap72); + set_interrupt_gate(cpu, 73, &trap73); + set_interrupt_gate(cpu, 74, &trap74); + set_interrupt_gate(cpu, 75, &trap75); + set_interrupt_gate(cpu, 76, &trap76); + set_interrupt_gate(cpu, 77, &trap77); + set_interrupt_gate(cpu, 78, &trap78); + set_interrupt_gate(cpu, 79, &trap79); + set_interrupt_gate(cpu, 80, &trap80); + set_interrupt_gate(cpu, 81, &trap81); + set_interrupt_gate(cpu, 82, &trap82); + set_interrupt_gate(cpu, 83, &trap83); + set_interrupt_gate(cpu, 84, &trap84); + set_interrupt_gate(cpu, 85, &trap85); + set_interrupt_gate(cpu, 86, &trap86); + set_interrupt_gate(cpu, 87, &trap87); + set_interrupt_gate(cpu, 88, &trap88); + set_interrupt_gate(cpu, 89, &trap89); + set_interrupt_gate(cpu, 90, &trap90); + set_interrupt_gate(cpu, 91, &trap91); + set_interrupt_gate(cpu, 92, &trap92); + set_interrupt_gate(cpu, 93, &trap93); + set_interrupt_gate(cpu, 94, &trap94); + set_interrupt_gate(cpu, 95, &trap95); + set_interrupt_gate(cpu, 96, &trap96); + set_interrupt_gate(cpu, 97, &trap97); + + set_trap_gate(cpu, 98, &trap98); // for performance testing only + set_trap_gate(cpu, 99, &trap99); // syscall interrupt + + // configurable msi or msi-x interrupts + set_interrupt_gate(cpu, 100, &trap100); + set_interrupt_gate(cpu, 101, &trap101); + set_interrupt_gate(cpu, 102, &trap102); + set_interrupt_gate(cpu, 103, &trap103); + set_interrupt_gate(cpu, 104, &trap104); + set_interrupt_gate(cpu, 105, &trap105); + set_interrupt_gate(cpu, 106, &trap106); + set_interrupt_gate(cpu, 107, &trap107); + set_interrupt_gate(cpu, 108, &trap108); + set_interrupt_gate(cpu, 109, &trap109); + set_interrupt_gate(cpu, 110, &trap110); + set_interrupt_gate(cpu, 111, &trap111); + set_interrupt_gate(cpu, 112, &trap112); + set_interrupt_gate(cpu, 113, &trap113); + set_interrupt_gate(cpu, 114, &trap114); + set_interrupt_gate(cpu, 115, &trap115); + set_interrupt_gate(cpu, 116, &trap116); + set_interrupt_gate(cpu, 117, &trap117); + set_interrupt_gate(cpu, 118, &trap118); + set_interrupt_gate(cpu, 119, &trap119); + set_interrupt_gate(cpu, 120, &trap120); + set_interrupt_gate(cpu, 121, &trap121); + set_interrupt_gate(cpu, 122, &trap122); + set_interrupt_gate(cpu, 123, &trap123); + set_interrupt_gate(cpu, 124, &trap124); + set_interrupt_gate(cpu, 125, &trap125); + set_interrupt_gate(cpu, 126, &trap126); + set_interrupt_gate(cpu, 127, &trap127); + set_interrupt_gate(cpu, 128, &trap128); + set_interrupt_gate(cpu, 129, &trap129); + set_interrupt_gate(cpu, 130, &trap130); + set_interrupt_gate(cpu, 131, &trap131); + set_interrupt_gate(cpu, 132, &trap132); + set_interrupt_gate(cpu, 133, &trap133); + set_interrupt_gate(cpu, 134, &trap134); + set_interrupt_gate(cpu, 135, &trap135); + set_interrupt_gate(cpu, 136, &trap136); + set_interrupt_gate(cpu, 137, &trap137); + set_interrupt_gate(cpu, 138, &trap138); + set_interrupt_gate(cpu, 139, &trap139); + set_interrupt_gate(cpu, 140, &trap140); + set_interrupt_gate(cpu, 141, &trap141); + set_interrupt_gate(cpu, 142, &trap142); + set_interrupt_gate(cpu, 143, &trap143); + set_interrupt_gate(cpu, 144, &trap144); + set_interrupt_gate(cpu, 145, &trap145); + set_interrupt_gate(cpu, 146, &trap146); + set_interrupt_gate(cpu, 147, &trap147); + set_interrupt_gate(cpu, 148, &trap148); + set_interrupt_gate(cpu, 149, &trap149); + set_interrupt_gate(cpu, 150, &trap150); + set_interrupt_gate(cpu, 151, &trap151); + set_interrupt_gate(cpu, 152, &trap152); + set_interrupt_gate(cpu, 153, &trap153); + set_interrupt_gate(cpu, 154, &trap154); + set_interrupt_gate(cpu, 155, &trap155); + set_interrupt_gate(cpu, 156, &trap156); + set_interrupt_gate(cpu, 157, &trap157); + set_interrupt_gate(cpu, 158, &trap158); + set_interrupt_gate(cpu, 159, &trap159); + set_interrupt_gate(cpu, 160, &trap160); + set_interrupt_gate(cpu, 161, &trap161); + set_interrupt_gate(cpu, 162, &trap162); + set_interrupt_gate(cpu, 163, &trap163); + set_interrupt_gate(cpu, 164, &trap164); + set_interrupt_gate(cpu, 165, &trap165); + set_interrupt_gate(cpu, 166, &trap166); + set_interrupt_gate(cpu, 167, &trap167); + set_interrupt_gate(cpu, 168, &trap168); + set_interrupt_gate(cpu, 169, &trap169); + set_interrupt_gate(cpu, 170, &trap170); + set_interrupt_gate(cpu, 171, &trap171); + set_interrupt_gate(cpu, 172, &trap172); + set_interrupt_gate(cpu, 173, &trap173); + set_interrupt_gate(cpu, 174, &trap174); + set_interrupt_gate(cpu, 175, &trap175); + set_interrupt_gate(cpu, 176, &trap176); + set_interrupt_gate(cpu, 177, &trap177); + set_interrupt_gate(cpu, 178, &trap178); + set_interrupt_gate(cpu, 179, &trap179); + set_interrupt_gate(cpu, 180, &trap180); + set_interrupt_gate(cpu, 181, &trap181); + set_interrupt_gate(cpu, 182, &trap182); + set_interrupt_gate(cpu, 183, &trap183); + set_interrupt_gate(cpu, 184, &trap184); + set_interrupt_gate(cpu, 185, &trap185); + set_interrupt_gate(cpu, 186, &trap186); + set_interrupt_gate(cpu, 187, &trap187); + set_interrupt_gate(cpu, 188, &trap188); + set_interrupt_gate(cpu, 189, &trap189); + set_interrupt_gate(cpu, 190, &trap190); + set_interrupt_gate(cpu, 191, &trap191); + set_interrupt_gate(cpu, 192, &trap192); + set_interrupt_gate(cpu, 193, &trap193); + set_interrupt_gate(cpu, 194, &trap194); + set_interrupt_gate(cpu, 195, &trap195); + set_interrupt_gate(cpu, 196, &trap196); + set_interrupt_gate(cpu, 197, &trap197); + set_interrupt_gate(cpu, 198, &trap198); + set_interrupt_gate(cpu, 199, &trap199); + set_interrupt_gate(cpu, 200, &trap200); + set_interrupt_gate(cpu, 201, &trap201); + set_interrupt_gate(cpu, 202, &trap202); + set_interrupt_gate(cpu, 203, &trap203); + set_interrupt_gate(cpu, 204, &trap204); + set_interrupt_gate(cpu, 205, &trap205); + set_interrupt_gate(cpu, 206, &trap206); + set_interrupt_gate(cpu, 207, &trap207); + set_interrupt_gate(cpu, 208, &trap208); + set_interrupt_gate(cpu, 209, &trap209); + set_interrupt_gate(cpu, 210, &trap210); + set_interrupt_gate(cpu, 211, &trap211); + set_interrupt_gate(cpu, 212, &trap212); + set_interrupt_gate(cpu, 213, &trap213); + set_interrupt_gate(cpu, 214, &trap214); + set_interrupt_gate(cpu, 215, &trap215); + set_interrupt_gate(cpu, 216, &trap216); + set_interrupt_gate(cpu, 217, &trap217); + set_interrupt_gate(cpu, 218, &trap218); + set_interrupt_gate(cpu, 219, &trap219); + set_interrupt_gate(cpu, 220, &trap220); + set_interrupt_gate(cpu, 221, &trap221); + set_interrupt_gate(cpu, 222, &trap222); + set_interrupt_gate(cpu, 223, &trap223); + set_interrupt_gate(cpu, 224, &trap224); + set_interrupt_gate(cpu, 225, &trap225); + set_interrupt_gate(cpu, 226, &trap226); + set_interrupt_gate(cpu, 227, &trap227); + set_interrupt_gate(cpu, 228, &trap228); + set_interrupt_gate(cpu, 229, &trap229); + set_interrupt_gate(cpu, 230, &trap230); + set_interrupt_gate(cpu, 231, &trap231); + set_interrupt_gate(cpu, 232, &trap232); + set_interrupt_gate(cpu, 233, &trap233); + set_interrupt_gate(cpu, 234, &trap234); + set_interrupt_gate(cpu, 235, &trap235); + set_interrupt_gate(cpu, 236, &trap236); + set_interrupt_gate(cpu, 237, &trap237); + set_interrupt_gate(cpu, 238, &trap238); + set_interrupt_gate(cpu, 239, &trap239); + set_interrupt_gate(cpu, 240, &trap240); + set_interrupt_gate(cpu, 241, &trap241); + set_interrupt_gate(cpu, 242, &trap242); + set_interrupt_gate(cpu, 243, &trap243); + set_interrupt_gate(cpu, 244, &trap244); + set_interrupt_gate(cpu, 245, &trap245); + set_interrupt_gate(cpu, 246, &trap246); + set_interrupt_gate(cpu, 247, &trap247); + set_interrupt_gate(cpu, 248, &trap248); + set_interrupt_gate(cpu, 249, &trap249); + set_interrupt_gate(cpu, 250, &trap250); + + // smp / apic local interrupts + set_interrupt_gate(cpu, 251, &trap251); + set_interrupt_gate(cpu, 252, &trap252); + set_interrupt_gate(cpu, 253, &trap253); + set_interrupt_gate(cpu, 254, &trap254); + set_interrupt_gate(cpu, 255, &trap255); + + load_idt(cpu); } // #pragma mark - +void +x86_descriptors_preboot_init_percpu(kernel_args* args, int cpu) +{ + init_idt_percpu(args, cpu); + init_gdt_percpu(args, cpu); +} + + void x86_descriptors_init(kernel_args* args) { - uint32 i; - interrupt_handler_function** table; - - // Get the GDT and boot CPU IDT set up by the boot loader. - gGDT = (segment_descriptor*)args->arch_args.vir_gdt; - sIDTs[0] = (interrupt_descriptor *)(addr_t)args->arch_args.vir_idt; - - set_interrupt_gate(0, 0, &trap0); - set_interrupt_gate(0, 1, &trap1); - set_interrupt_gate(0, 2, &trap2); - set_trap_gate(0, 3, &trap3); - set_interrupt_gate(0, 4, &trap4); - set_interrupt_gate(0, 5, &trap5); - set_interrupt_gate(0, 6, &trap6); - set_interrupt_gate(0, 7, &trap7); - // trap8 (double fault) is set in init_double_fault(). - set_interrupt_gate(0, 9, &trap9); - set_interrupt_gate(0, 10, &trap10); - set_interrupt_gate(0, 11, &trap11); - set_interrupt_gate(0, 12, &trap12); - set_interrupt_gate(0, 13, &trap13); - set_interrupt_gate(0, 14, &trap14); -// set_interrupt_gate(0, 15, &trap15); - set_interrupt_gate(0, 16, &trap16); - set_interrupt_gate(0, 17, &trap17); - set_interrupt_gate(0, 18, &trap18); - set_interrupt_gate(0, 19, &trap19); - - // legacy or ioapic interrupts - set_interrupt_gate(0, 32, &trap32); - set_interrupt_gate(0, 33, &trap33); - set_interrupt_gate(0, 34, &trap34); - set_interrupt_gate(0, 35, &trap35); - set_interrupt_gate(0, 36, &trap36); - set_interrupt_gate(0, 37, &trap37); - set_interrupt_gate(0, 38, &trap38); - set_interrupt_gate(0, 39, &trap39); - set_interrupt_gate(0, 40, &trap40); - set_interrupt_gate(0, 41, &trap41); - set_interrupt_gate(0, 42, &trap42); - set_interrupt_gate(0, 43, &trap43); - set_interrupt_gate(0, 44, &trap44); - set_interrupt_gate(0, 45, &trap45); - set_interrupt_gate(0, 46, &trap46); - set_interrupt_gate(0, 47, &trap47); - - // additional ioapic interrupts - set_interrupt_gate(0, 48, &trap48); - set_interrupt_gate(0, 49, &trap49); - set_interrupt_gate(0, 50, &trap50); - set_interrupt_gate(0, 51, &trap51); - set_interrupt_gate(0, 52, &trap52); - set_interrupt_gate(0, 53, &trap53); - set_interrupt_gate(0, 54, &trap54); - set_interrupt_gate(0, 55, &trap55); - - // configurable msi or msi-x interrupts - set_interrupt_gate(0, 56, &trap56); - set_interrupt_gate(0, 57, &trap57); - set_interrupt_gate(0, 58, &trap58); - set_interrupt_gate(0, 59, &trap59); - set_interrupt_gate(0, 60, &trap60); - set_interrupt_gate(0, 61, &trap61); - set_interrupt_gate(0, 62, &trap62); - set_interrupt_gate(0, 63, &trap63); - set_interrupt_gate(0, 64, &trap64); - set_interrupt_gate(0, 65, &trap65); - set_interrupt_gate(0, 66, &trap66); - set_interrupt_gate(0, 67, &trap67); - set_interrupt_gate(0, 68, &trap68); - set_interrupt_gate(0, 69, &trap69); - set_interrupt_gate(0, 70, &trap70); - set_interrupt_gate(0, 71, &trap71); - set_interrupt_gate(0, 72, &trap72); - set_interrupt_gate(0, 73, &trap73); - set_interrupt_gate(0, 74, &trap74); - set_interrupt_gate(0, 75, &trap75); - set_interrupt_gate(0, 76, &trap76); - set_interrupt_gate(0, 77, &trap77); - set_interrupt_gate(0, 78, &trap78); - set_interrupt_gate(0, 79, &trap79); - set_interrupt_gate(0, 80, &trap80); - set_interrupt_gate(0, 81, &trap81); - set_interrupt_gate(0, 82, &trap82); - set_interrupt_gate(0, 83, &trap83); - set_interrupt_gate(0, 84, &trap84); - set_interrupt_gate(0, 85, &trap85); - set_interrupt_gate(0, 86, &trap86); - set_interrupt_gate(0, 87, &trap87); - set_interrupt_gate(0, 88, &trap88); - set_interrupt_gate(0, 89, &trap89); - set_interrupt_gate(0, 90, &trap90); - set_interrupt_gate(0, 91, &trap91); - set_interrupt_gate(0, 92, &trap92); - set_interrupt_gate(0, 93, &trap93); - set_interrupt_gate(0, 94, &trap94); - set_interrupt_gate(0, 95, &trap95); - set_interrupt_gate(0, 96, &trap96); - set_interrupt_gate(0, 97, &trap97); - - set_trap_gate(0, 98, &trap98); // for performance testing only - set_trap_gate(0, 99, &trap99); // syscall interrupt - reserve_io_interrupt_vectors(2, 98, INTERRUPT_TYPE_SYSCALL); - // configurable msi or msi-x interrupts - set_interrupt_gate(0, 100, &trap100); - set_interrupt_gate(0, 101, &trap101); - set_interrupt_gate(0, 102, &trap102); - set_interrupt_gate(0, 103, &trap103); - set_interrupt_gate(0, 104, &trap104); - set_interrupt_gate(0, 105, &trap105); - set_interrupt_gate(0, 106, &trap106); - set_interrupt_gate(0, 107, &trap107); - set_interrupt_gate(0, 108, &trap108); - set_interrupt_gate(0, 109, &trap109); - set_interrupt_gate(0, 110, &trap110); - set_interrupt_gate(0, 111, &trap111); - set_interrupt_gate(0, 112, &trap112); - set_interrupt_gate(0, 113, &trap113); - set_interrupt_gate(0, 114, &trap114); - set_interrupt_gate(0, 115, &trap115); - set_interrupt_gate(0, 116, &trap116); - set_interrupt_gate(0, 117, &trap117); - set_interrupt_gate(0, 118, &trap118); - set_interrupt_gate(0, 119, &trap119); - set_interrupt_gate(0, 120, &trap120); - set_interrupt_gate(0, 121, &trap121); - set_interrupt_gate(0, 122, &trap122); - set_interrupt_gate(0, 123, &trap123); - set_interrupt_gate(0, 124, &trap124); - set_interrupt_gate(0, 125, &trap125); - set_interrupt_gate(0, 126, &trap126); - set_interrupt_gate(0, 127, &trap127); - set_interrupt_gate(0, 128, &trap128); - set_interrupt_gate(0, 129, &trap129); - set_interrupt_gate(0, 130, &trap130); - set_interrupt_gate(0, 131, &trap131); - set_interrupt_gate(0, 132, &trap132); - set_interrupt_gate(0, 133, &trap133); - set_interrupt_gate(0, 134, &trap134); - set_interrupt_gate(0, 135, &trap135); - set_interrupt_gate(0, 136, &trap136); - set_interrupt_gate(0, 137, &trap137); - set_interrupt_gate(0, 138, &trap138); - set_interrupt_gate(0, 139, &trap139); - set_interrupt_gate(0, 140, &trap140); - set_interrupt_gate(0, 141, &trap141); - set_interrupt_gate(0, 142, &trap142); - set_interrupt_gate(0, 143, &trap143); - set_interrupt_gate(0, 144, &trap144); - set_interrupt_gate(0, 145, &trap145); - set_interrupt_gate(0, 146, &trap146); - set_interrupt_gate(0, 147, &trap147); - set_interrupt_gate(0, 148, &trap148); - set_interrupt_gate(0, 149, &trap149); - set_interrupt_gate(0, 150, &trap150); - set_interrupt_gate(0, 151, &trap151); - set_interrupt_gate(0, 152, &trap152); - set_interrupt_gate(0, 153, &trap153); - set_interrupt_gate(0, 154, &trap154); - set_interrupt_gate(0, 155, &trap155); - set_interrupt_gate(0, 156, &trap156); - set_interrupt_gate(0, 157, &trap157); - set_interrupt_gate(0, 158, &trap158); - set_interrupt_gate(0, 159, &trap159); - set_interrupt_gate(0, 160, &trap160); - set_interrupt_gate(0, 161, &trap161); - set_interrupt_gate(0, 162, &trap162); - set_interrupt_gate(0, 163, &trap163); - set_interrupt_gate(0, 164, &trap164); - set_interrupt_gate(0, 165, &trap165); - set_interrupt_gate(0, 166, &trap166); - set_interrupt_gate(0, 167, &trap167); - set_interrupt_gate(0, 168, &trap168); - set_interrupt_gate(0, 169, &trap169); - set_interrupt_gate(0, 170, &trap170); - set_interrupt_gate(0, 171, &trap171); - set_interrupt_gate(0, 172, &trap172); - set_interrupt_gate(0, 173, &trap173); - set_interrupt_gate(0, 174, &trap174); - set_interrupt_gate(0, 175, &trap175); - set_interrupt_gate(0, 176, &trap176); - set_interrupt_gate(0, 177, &trap177); - set_interrupt_gate(0, 178, &trap178); - set_interrupt_gate(0, 179, &trap179); - set_interrupt_gate(0, 180, &trap180); - set_interrupt_gate(0, 181, &trap181); - set_interrupt_gate(0, 182, &trap182); - set_interrupt_gate(0, 183, &trap183); - set_interrupt_gate(0, 184, &trap184); - set_interrupt_gate(0, 185, &trap185); - set_interrupt_gate(0, 186, &trap186); - set_interrupt_gate(0, 187, &trap187); - set_interrupt_gate(0, 188, &trap188); - set_interrupt_gate(0, 189, &trap189); - set_interrupt_gate(0, 190, &trap190); - set_interrupt_gate(0, 191, &trap191); - set_interrupt_gate(0, 192, &trap192); - set_interrupt_gate(0, 193, &trap193); - set_interrupt_gate(0, 194, &trap194); - set_interrupt_gate(0, 195, &trap195); - set_interrupt_gate(0, 196, &trap196); - set_interrupt_gate(0, 197, &trap197); - set_interrupt_gate(0, 198, &trap198); - set_interrupt_gate(0, 199, &trap199); - set_interrupt_gate(0, 200, &trap200); - set_interrupt_gate(0, 201, &trap201); - set_interrupt_gate(0, 202, &trap202); - set_interrupt_gate(0, 203, &trap203); - set_interrupt_gate(0, 204, &trap204); - set_interrupt_gate(0, 205, &trap205); - set_interrupt_gate(0, 206, &trap206); - set_interrupt_gate(0, 207, &trap207); - set_interrupt_gate(0, 208, &trap208); - set_interrupt_gate(0, 209, &trap209); - set_interrupt_gate(0, 210, &trap210); - set_interrupt_gate(0, 211, &trap211); - set_interrupt_gate(0, 212, &trap212); - set_interrupt_gate(0, 213, &trap213); - set_interrupt_gate(0, 214, &trap214); - set_interrupt_gate(0, 215, &trap215); - set_interrupt_gate(0, 216, &trap216); - set_interrupt_gate(0, 217, &trap217); - set_interrupt_gate(0, 218, &trap218); - set_interrupt_gate(0, 219, &trap219); - set_interrupt_gate(0, 220, &trap220); - set_interrupt_gate(0, 221, &trap221); - set_interrupt_gate(0, 222, &trap222); - set_interrupt_gate(0, 223, &trap223); - set_interrupt_gate(0, 224, &trap224); - set_interrupt_gate(0, 225, &trap225); - set_interrupt_gate(0, 226, &trap226); - set_interrupt_gate(0, 227, &trap227); - set_interrupt_gate(0, 228, &trap228); - set_interrupt_gate(0, 229, &trap229); - set_interrupt_gate(0, 230, &trap230); - set_interrupt_gate(0, 231, &trap231); - set_interrupt_gate(0, 232, &trap232); - set_interrupt_gate(0, 233, &trap233); - set_interrupt_gate(0, 234, &trap234); - set_interrupt_gate(0, 235, &trap235); - set_interrupt_gate(0, 236, &trap236); - set_interrupt_gate(0, 237, &trap237); - set_interrupt_gate(0, 238, &trap238); - set_interrupt_gate(0, 239, &trap239); - set_interrupt_gate(0, 240, &trap240); - set_interrupt_gate(0, 241, &trap241); - set_interrupt_gate(0, 242, &trap242); - set_interrupt_gate(0, 243, &trap243); - set_interrupt_gate(0, 244, &trap244); - set_interrupt_gate(0, 245, &trap245); - set_interrupt_gate(0, 246, &trap246); - set_interrupt_gate(0, 247, &trap247); - set_interrupt_gate(0, 248, &trap248); - set_interrupt_gate(0, 249, &trap249); - set_interrupt_gate(0, 250, &trap250); - - // smp / apic local interrupts - set_interrupt_gate(0, 251, &trap251); - set_interrupt_gate(0, 252, &trap252); - set_interrupt_gate(0, 253, &trap253); - set_interrupt_gate(0, 254, &trap254); - set_interrupt_gate(0, 255, &trap255); - // init interrupt handler table - table = gInterruptHandlerTable; + interrupt_handler_function** table = gInterruptHandlerTable; // defaults + uint32 i; for (i = 0; i < ARCH_INTERRUPT_BASE; i++) table[i] = x86_invalid_exception; for (i = ARCH_INTERRUPT_BASE; i < kInterruptHandlerTableSize; i++) @@ -505,90 +595,14 @@ x86_descriptors_init(kernel_args* args) void -x86_descriptors_init_percpu(kernel_args* args, int cpu) +x86_descriptors_init_percpu(kernel_args* /* args */, int /* cpu */) { - // load the TSS for this cpu - // note the main cpu gets initialized in x86_descriptors_init_post_vm() - if (cpu != 0) { - load_tss(cpu); - - // set the IDT - struct { - uint16 limit; - void* address; - } _PACKED descriptor = { - 256 * 8 - 1, // 256 descriptors, 8 bytes each (-1 for "limit") - sIDTs[cpu] - }; - - asm volatile("lidt %0" : : "m"(descriptor)); - } } status_t -x86_descriptors_init_post_vm(kernel_args* args) +x86_descriptors_init_post_vm(kernel_args* /* args */) { - uint32 i; - - // account for the segment descriptors - create_area("gdt", (void **)&gGDT, B_EXACT_ADDRESS, B_PAGE_SIZE, - B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - - // create IDT area for the boot CPU - area_id area = create_area("idt", (void**)&sIDTs[0], B_EXACT_ADDRESS, - B_PAGE_SIZE, B_ALREADY_WIRED, B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA); - if (area < B_OK) - return area; - - // create IDTs for the off-boot CPU - size_t idtSize = 256 * 8; - // 256 8 bytes-sized descriptors - if (args->num_cpus > 0) { - size_t areaSize = ROUNDUP(args->num_cpus * idtSize, B_PAGE_SIZE); - interrupt_descriptor* idt; - virtual_address_restrictions virtualRestrictions = {}; - virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS; - physical_address_restrictions physicalRestrictions = {}; - area = create_area_etc(B_SYSTEM_TEAM, "idt", areaSize, B_CONTIGUOUS, - B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, - 0, &virtualRestrictions, &physicalRestrictions, (void**)&idt); - if (area < 0) - return area; - - for (i = 1; i < args->num_cpus; i++) { - sIDTs[i] = idt; - memcpy(idt, sIDTs[0], idtSize); - idt += 256; - // The CPU's IDTR will be set in arch_cpu_init_percpu(). - } - } - - // setup task-state segments - for (i = 0; i < args->num_cpus; i++) { - // initialize the regular and double fault tss stored in the per-cpu - // structure - memset(&gCPU[i].arch.tss, 0, sizeof(struct tss)); - gCPU[i].arch.tss.ss0 = KERNEL_DATA_SEG; - gCPU[i].arch.tss.io_map_base = sizeof(struct tss); - - // add TSS descriptor for this new TSS - set_tss_descriptor(&gGDT[TSS_SEGMENT(i)], (addr_t)&gCPU[i].arch.tss, - sizeof(struct tss)); - - // initialize the double fault tss - init_double_fault(i); - } - - // set the current hardware task on cpu 0 - load_tss(0); - - // setup TLS descriptors (one for every CPU) - - for (i = 0; i < args->num_cpus; i++) { - set_segment_descriptor(&gGDT[TLS_BASE_SEGMENT + i], 0, TLS_SIZE, - DT_DATA_WRITEABLE, DPL_USER); - } - return B_OK; } + diff --git a/src/system/kernel/arch/x86/32/interrupts.S b/src/system/kernel/arch/x86/32/interrupts.S index 912c583cb9..917ca79cf9 100644 --- a/src/system/kernel/arch/x86/32/interrupts.S +++ b/src/system/kernel/arch/x86/32/interrupts.S @@ -223,7 +223,7 @@ FUNCTION(double_fault): pushl $-1; // user-ss pushl $-1; // user-esp pushl $-1; // flags - pushl $KERNEL_CODE_SEG; // cs + pushl $KERNEL_CODE_SELECTOR // cs pushl $-1; // eip pushl $0; // error-code pushl $8; @@ -516,7 +516,7 @@ STATIC_FUNCTION(int_bottom): // exception. orl $0x10000, IFRAME_flags(%ebp); - cmp $USER_CODE_SEG, IFRAME_cs(%ebp) // user mode + cmpw $USER_CODE_SELECTOR, IFRAME_cs(%ebp) // user mode je int_bottom_user // We need to recheck user mode using the thread's in_kernel flag, since @@ -540,7 +540,7 @@ FUNCTION_END(int_bottom) STATIC_FUNCTION(int_bottom_user): - movl $KERNEL_DATA_SEG,%eax + movl $KERNEL_DATA_SELECTOR, %eax cld movl %eax,%ds movl %eax,%es @@ -564,7 +564,7 @@ STATIC_FUNCTION(int_bottom_user): // were already/still prepared for userland), since the iframe in this case // will be a kernel iframe and e.g. trying to set up a signal stack will not // be a very healthy endeavor. - cmp $USER_CODE_SEG, IFRAME_cs(%ebp) + cmpw $USER_CODE_SELECTOR, IFRAME_cs(%ebp) jne 1f testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ @@ -603,7 +603,7 @@ STATIC_FUNCTION(handle_syscall): // save %eax, the number of the syscall movl %eax, %esi - movl $KERNEL_DATA_SEG,%eax + movl $KERNEL_DATA_SELECTOR, %eax cld movl %eax,%ds movl %eax,%es @@ -759,11 +759,11 @@ FUNCTION(x86_sysenter): movl %dr3, %edx // push the iframe - pushl $USER_DATA_SEG // user_ss + pushl $USER_DATA_SELECTOR // user_ss pushl %ecx // user_esp pushfl // eflags orl $(1 << 9), (%esp) // set the IF (interrupts) bit - pushl $USER_CODE_SEG // user cs + pushl $USER_CODE_SELECTOR // user cs // user_eip movl THREAD_team(%edx), %edx diff --git a/src/system/kernel/arch/x86/32/syscalls.cpp b/src/system/kernel/arch/x86/32/syscalls.cpp index 5b007e61e1..c3ca4e1a24 100644 --- a/src/system/kernel/arch/x86/32/syscalls.cpp +++ b/src/system/kernel/arch/x86/32/syscalls.cpp @@ -55,7 +55,7 @@ set_intel_syscall_stack(addr_t stackTop) static void init_intel_syscall_registers(void* dummy, int cpuNum) { - x86_write_msr(IA32_MSR_SYSENTER_CS, KERNEL_CODE_SEG); + x86_write_msr(IA32_MSR_SYSENTER_CS, KERNEL_CODE_SELECTOR); x86_write_msr(IA32_MSR_SYSENTER_ESP, 0); x86_write_msr(IA32_MSR_SYSENTER_EIP, (addr_t)x86_sysenter); diff --git a/src/system/kernel/arch/x86/32/thread.cpp b/src/system/kernel/arch/x86/32/thread.cpp index 7bed6a0f4d..a62575590b 100644 --- a/src/system/kernel/arch/x86/32/thread.cpp +++ b/src/system/kernel/arch/x86/32/thread.cpp @@ -69,7 +69,6 @@ class RestartSyscall : public AbstractTraceEntry { // from arch_cpu.cpp extern bool gHasSSE; -extern segment_descriptor* gGDT; static struct arch_thread sInitialState _ALIGNED(16); // the fpu_state must be aligned on a 16 byte boundary, so that fxsave can use it @@ -103,10 +102,10 @@ x86_restart_syscall(struct iframe* frame) void x86_set_tls_context(Thread *thread) { - int entry = smp_get_current_cpu() + TLS_BASE_SEGMENT; - - set_segment_descriptor_base(&gGDT[entry], thread->user_local_storage); - set_fs_register((entry << 3) | DPL_USER); + segment_descriptor* gdt = get_gdt(smp_get_current_cpu()); + set_segment_descriptor_base(&gdt[USER_TLS_SEGMENT], + thread->user_local_storage); + set_fs_register((USER_TLS_SEGMENT << 3) | DPL_USER); } @@ -197,7 +196,7 @@ arch_thread_init_kthread_stack(Thread* thread, void* _stack, void* _stackTop, // save the stack position thread->arch_info.current_stack.esp = stackTop; - thread->arch_info.current_stack.ss = (addr_t*)KERNEL_DATA_SEG; + thread->arch_info.current_stack.ss = (addr_t*)KERNEL_DATA_SELECTOR; } @@ -242,16 +241,16 @@ arch_thread_enter_userspace(Thread* thread, addr_t entry, void* args1, // prepare the user iframe iframe frame = {}; frame.type = IFRAME_TYPE_SYSCALL; - frame.gs = USER_DATA_SEG; + frame.gs = USER_DATA_SELECTOR; // frame.fs not used, we call x86_set_tls_context() on context switch - frame.es = USER_DATA_SEG; - frame.ds = USER_DATA_SEG; + frame.es = USER_DATA_SELECTOR; + frame.ds = USER_DATA_SELECTOR; frame.ip = entry; - frame.cs = USER_CODE_SEG; + frame.cs = USER_CODE_SELECTOR; frame.flags = X86_EFLAGS_RESERVED1 | X86_EFLAGS_INTERRUPT | (3 << X86_EFLAGS_IO_PRIVILEG_LEVEL_SHIFT); frame.user_sp = stackTop; - frame.user_ss = USER_DATA_SEG; + frame.user_ss = USER_DATA_SELECTOR; // return to userland x86_initial_return_to_userland(thread, &frame); diff --git a/src/system/kernel/arch/x86/64/descriptors.cpp b/src/system/kernel/arch/x86/64/descriptors.cpp index 955f96c1e4..3e6878eb64 100644 --- a/src/system/kernel/arch/x86/64/descriptors.cpp +++ b/src/system/kernel/arch/x86/64/descriptors.cpp @@ -60,6 +60,12 @@ x86_64_general_protection_fault(iframe* frame) // #pragma mark - +void +x86_descriptors_preboot_init_percpu(kernel_args* /* args */, int /* cpu */) +{ +} + + void x86_descriptors_init(kernel_args* args) { @@ -83,7 +89,7 @@ x86_descriptors_init(kernel_args* args) uint32 dpl = (i == 3) ? DPL_USER : DPL_KERNEL; set_interrupt_descriptor(&sIDT[i], (addr_t)&isr_array[i], - GATE_INTERRUPT, KERNEL_CODE_SEG, dpl, ist); + GATE_INTERRUPT, KERNEL_CODE_SELECTOR, dpl, ist); } // Initialize the interrupt handler table. diff --git a/src/system/kernel/arch/x86/64/interrupts.S b/src/system/kernel/arch/x86/64/interrupts.S index 71a0d106db..db73a4fba4 100644 --- a/src/system/kernel/arch/x86/64/interrupts.S +++ b/src/system/kernel/arch/x86/64/interrupts.S @@ -302,10 +302,10 @@ FUNCTION(x86_64_syscall_entry): movq %gs:ARCH_THREAD_syscall_rsp, %rsp // Set up an iframe on the stack (R11 = saved RFLAGS, RCX = saved RIP). - push $USER_DATA_SEG // ss + push $USER_DATA_SELECTOR // ss push %gs:ARCH_THREAD_user_rsp // rsp push %r11 // flags - push $USER_CODE_SEG // cs + push $USER_CODE_SELECTOR // cs push %rcx // ip push $0 // error_code push $99 // vector diff --git a/src/system/kernel/arch/x86/64/syscalls.cpp b/src/system/kernel/arch/x86/64/syscalls.cpp index 4407498aa2..174755f5a0 100644 --- a/src/system/kernel/arch/x86/64/syscalls.cpp +++ b/src/system/kernel/arch/x86/64/syscalls.cpp @@ -38,12 +38,12 @@ init_syscall_registers(void* dummy, int cpuNum) // - CS is set to IA32_STAR[63:48] + 16 // - SS is set to IA32_STAR[63:48] + 8 // From this we get: - // - Entry CS = KERNEL_CODE_SEG - // - Entry SS = KERNEL_CODE_SEG + 8 = KERNEL_DATA_SEG - // - Return CS = KERNEL_DATA_SEG + 16 = USER_CODE_SEG - // - Return SS = KERNEL_DATA_SEG + 8 = USER_DATA_SEG - x86_write_msr(IA32_MSR_STAR, ((uint64)(KERNEL_DATA_SEG | 3) << 48) - | ((uint64)KERNEL_CODE_SEG << 32)); + // - Entry CS = KERNEL_CODE_SELECTOR + // - Entry SS = KERNEL_CODE_SELECTOR + 8 = KERNEL_DATA_SELECTOR + // - Return CS = KERNEL_DATA_SELECTOR + 16 = USER_CODE_SELECTOR + // - Return SS = KERNEL_DATA_SELECTOR + 8 = USER_DATA_SLECTORG + x86_write_msr(IA32_MSR_STAR, ((uint64)(KERNEL_DATA_SELECTOR | 3) << 48) + | ((uint64)KERNEL_CODE_SELECTOR << 32)); } diff --git a/src/system/kernel/arch/x86/64/thread.cpp b/src/system/kernel/arch/x86/64/thread.cpp index 59db5028e4..5ee839a711 100644 --- a/src/system/kernel/arch/x86/64/thread.cpp +++ b/src/system/kernel/arch/x86/64/thread.cpp @@ -238,11 +238,11 @@ arch_thread_enter_userspace(Thread* thread, addr_t entry, void* args1, frame.si = (uint64)args2; frame.di = (uint64)args1; frame.ip = entry; - frame.cs = USER_CODE_SEG; + frame.cs = USER_CODE_SELECTOR; frame.flags = X86_EFLAGS_RESERVED1 | X86_EFLAGS_INTERRUPT | (3 << X86_EFLAGS_IO_PRIVILEG_LEVEL_SHIFT); frame.sp = stackTop; - frame.ss = USER_DATA_SEG; + frame.ss = USER_DATA_SELECTOR; // Return to userland. Never returns. x86_initial_return_to_userland(thread, &frame); diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 2046f2f176..f4cc5ffba3 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -1008,6 +1008,8 @@ arch_cpu_preboot_init_percpu(kernel_args* args, int cpu) x86_write_msr(IA32_MSR_TSC, 0); } + x86_descriptors_preboot_init_percpu(args, cpu); + return B_OK; } @@ -1049,7 +1051,6 @@ detect_amdc1e_noarat() status_t arch_cpu_init_percpu(kernel_args* args, int cpu) { - // Load descriptor tables for this CPU. x86_descriptors_init_percpu(args, cpu); detect_cpu(cpu); From a5b070f1fac87567bd1837c53bdd7d44f3a5d3c4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 17 Dec 2013 04:08:51 +0100 Subject: [PATCH 200/273] x86: Store pointer to the current thread in gs:0 Apparently, reading from dr3 is slower than reading from memory with cache hit. Also, depending on hypervisor configuration, accessing dr3 may cause a VM exit (and, at least on kvm, it does), what makes it much slower than a memory access even when there is a cache miss. --- .../private/kernel/arch/x86/32/descriptors.h | 2 ++ headers/private/kernel/arch/x86/arch_thread.h | 17 ++++--------- src/system/kernel/arch/x86/32/interrupts.S | 24 +++++++++++++------ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/headers/private/kernel/arch/x86/32/descriptors.h b/headers/private/kernel/arch/x86/32/descriptors.h index 3bdc8c3310..5308ce0547 100644 --- a/headers/private/kernel/arch/x86/32/descriptors.h +++ b/headers/private/kernel/arch/x86/32/descriptors.h @@ -38,6 +38,8 @@ #define USER_CODE_SELECTOR ((USER_CODE_SEGMENT << 3) | DPL_USER) #define USER_DATA_SELECTOR ((USER_DATA_SEGMENT << 3) | DPL_USER) +#define KERNEL_TLS_SELECTOR ((KERNEL_TLS_SEGMENT << 3) | DPL_KERNEL) + #ifndef _ASSEMBLER // this file can also be included from assembler as well diff --git a/headers/private/kernel/arch/x86/arch_thread.h b/headers/private/kernel/arch/x86/arch_thread.h index 0769195543..b7e1493c15 100644 --- a/headers/private/kernel/arch/x86/arch_thread.h +++ b/headers/private/kernel/arch/x86/arch_thread.h @@ -30,9 +30,6 @@ void x86_restart_syscall(struct iframe* frame); void x86_set_tls_context(Thread* thread); -#ifdef __x86_64__ - - static inline Thread* arch_thread_get_current_thread(void) { @@ -42,6 +39,9 @@ arch_thread_get_current_thread(void) } +#ifdef __x86_64__ + + static inline void arch_thread_set_current_thread(Thread* t) { @@ -59,18 +59,10 @@ arch_thread_set_current_thread(Thread* t) void arch_syscall_64_bit_return_value(void); -static inline Thread* -arch_thread_get_current_thread(void) -{ - Thread* t = (Thread*)x86_read_dr3(); - return t; -} - - static inline void arch_thread_set_current_thread(Thread* t) { - x86_write_dr3(t); + asm volatile("mov %0, %%gs:0" : : "r" (t)); } @@ -82,3 +74,4 @@ arch_thread_set_current_thread(Thread* t) #endif #endif /* _KERNEL_ARCH_x86_THREAD_H */ + diff --git a/src/system/kernel/arch/x86/32/interrupts.S b/src/system/kernel/arch/x86/32/interrupts.S index 917ca79cf9..d42f66ed34 100644 --- a/src/system/kernel/arch/x86/32/interrupts.S +++ b/src/system/kernel/arch/x86/32/interrupts.S @@ -159,7 +159,7 @@ \ /* restore pointers and clear fault handler */ \ movl %edx, %esi; /* syscall info pointer */ \ - movl %dr3, %edi; /* thread pointer */ \ + movl %gs:0, %edi; /* thread pointer */ \ movl $0, THREAD_fault_handler(%edi) #if SYSCALL_TRACING @@ -509,6 +509,9 @@ FUNCTION_END(trap14_double_fault) STATIC_FUNCTION(int_bottom): PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER) + movl $KERNEL_TLS_SELECTOR, %edx + movw %dx, %gs + movl %esp, %ebp // frame pointer is the iframe // Set the RF (resume flag) in EFLAGS. This prevents an instruction @@ -524,7 +527,7 @@ STATIC_FUNCTION(int_bottom): // so that we have to do it in the instruction before, thus opening a // window for an interrupt while still being in the kernel, but having set // up everything for userland already. - movl %dr3, %edi // thread pointer + movl %gs:0, %edi // thread pointer cmpb $0, THREAD_in_kernel(%edi) je int_bottom_user @@ -546,7 +549,7 @@ STATIC_FUNCTION(int_bottom_user): movl %eax,%es // disable breakpoints, if installed - movl %dr3, %edi // thread pointer + movl %gs:0, %edi // thread pointer cli // disable interrupts STOP_USER_DEBUGGING() @@ -600,6 +603,9 @@ FUNCTION_END(trap99) STATIC_FUNCTION(handle_syscall): + movl $KERNEL_TLS_SELECTOR, %edx + movw %dx, %gs + // save %eax, the number of the syscall movl %eax, %esi @@ -610,7 +616,7 @@ STATIC_FUNCTION(handle_syscall): lea 4(%esp), %ebp // skipping the return address, the stack // frame pointer is the iframe - movl %dr3, %edi // thread pointer + movl %gs:0, %edi // thread pointer // disable breakpoints, if installed cli // disable interrupts @@ -745,7 +751,7 @@ FUNCTION_END(handle_syscall) STATIC_FUNCTION(bad_syscall_params): // clear the fault handler and exit normally - movl %dr3, %edi + movl %gs:0, %edi movl $0, THREAD_fault_handler(%edi) jmp kernel_exit_work FUNCTION_END(bad_syscall_params) @@ -756,7 +762,11 @@ FUNCTION_END(handle_syscall) */ FUNCTION(x86_sysenter): // get the thread - movl %dr3, %edx + push %gs + movl $KERNEL_TLS_SELECTOR, %edx + movw %dx, %gs + movl %gs:0, %edx + pop %gs // push the iframe pushl $USER_DATA_SELECTOR // user_ss @@ -816,7 +826,7 @@ FUNCTION(x86_return_to_userland): movl %ebp, %esp // check, if any kernel exit work has to be done - movl %dr3, %edi + movl %gs:0, %edi testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \ | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED) \ , THREAD_flags(%edi) From 735f67481fa904ae6bb5d0763f515fd7b93d5529 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 17 Dec 2013 04:31:29 +0100 Subject: [PATCH 201/273] x86: Debugger can now use dr3 --- headers/private/kernel/arch/x86/arch_thread.h | 2 +- headers/private/kernel/arch/x86/arch_user_debugger.h | 7 +------ src/system/kernel/arch/x86/arch_debug.cpp | 4 ++-- src/system/kernel/arch/x86/arch_user_debugger.cpp | 3 --- 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_thread.h b/headers/private/kernel/arch/x86/arch_thread.h index b7e1493c15..3689d0b44b 100644 --- a/headers/private/kernel/arch/x86/arch_thread.h +++ b/headers/private/kernel/arch/x86/arch_thread.h @@ -62,7 +62,7 @@ void arch_syscall_64_bit_return_value(void); static inline void arch_thread_set_current_thread(Thread* t) { - asm volatile("mov %0, %%gs:0" : : "r" (t)); + asm volatile("mov %0, %%gs:0" : : "r" (t) : "memory"); } diff --git a/headers/private/kernel/arch/x86/arch_user_debugger.h b/headers/private/kernel/arch/x86/arch_user_debugger.h index 01a98e7639..c740c57dd4 100644 --- a/headers/private/kernel/arch/x86/arch_user_debugger.h +++ b/headers/private/kernel/arch/x86/arch_user_debugger.h @@ -9,12 +9,7 @@ #define ARCH_INIT_USER_DEBUG x86_init_user_debug // number of breakpoints the CPU supports -// On 32-bit, DR3 is used to hold the Thread*. -#ifdef __x86_64__ -# define X86_BREAKPOINT_COUNT 4 -#else -# define X86_BREAKPOINT_COUNT 3 -#endif +#define X86_BREAKPOINT_COUNT 4 // debug status register DR6 enum { diff --git a/src/system/kernel/arch/x86/arch_debug.cpp b/src/system/kernel/arch/x86/arch_debug.cpp index ea84e5c807..199f16f402 100644 --- a/src/system/kernel/arch/x86/arch_debug.cpp +++ b/src/system/kernel/arch/x86/arch_debug.cpp @@ -1202,13 +1202,13 @@ arch_debug_get_interrupt_pc(bool* _isSyscall) void arch_debug_unset_current_thread(void) { -#ifdef __x86_64__ // Can't just write 0 to the GS base, that will cause the read from %gs:0 // to fault. Instead point it at a NULL pointer, %gs:0 will get this value. static Thread* unsetThread = NULL; +#ifdef __x86_64__ x86_write_msr(IA32_MSR_GS_BASE, (addr_t)&unsetThread); #else - x86_write_dr3(NULL); + asm volatile("mov %0, %%gs:0" : : "r" (unsetThread) : "memory"); #endif } diff --git a/src/system/kernel/arch/x86/arch_user_debugger.cpp b/src/system/kernel/arch/x86/arch_user_debugger.cpp index 9c1a721b1e..516ed8f340 100644 --- a/src/system/kernel/arch/x86/arch_user_debugger.cpp +++ b/src/system/kernel/arch/x86/arch_user_debugger.cpp @@ -201,10 +201,7 @@ install_breakpoints(const arch_team_debug_info& teamInfo) asm("mov %0, %%dr0" : : "r"(teamInfo.breakpoints[0].address)); asm("mov %0, %%dr1" : : "r"(teamInfo.breakpoints[1].address)); asm("mov %0, %%dr2" : : "r"(teamInfo.breakpoints[2].address)); -#ifdef __x86_64__ asm("mov %0, %%dr3" : : "r"(teamInfo.breakpoints[3].address)); - // DR3 is used to hold the current Thread* on 32. -#endif // enable breakpoints asm("mov %0, %%dr7" : : "r"(teamInfo.dr7)); From 4fcbac58de122ff3313cb0bfe9115314d28bf219 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 17 Dec 2013 22:08:18 +0100 Subject: [PATCH 202/273] kernel: Make get_cpu_topology() return a pointer to const The client code is not supposed to change the topology info. It would be also nice if cpu_topology_node::children was an array of pointers to const but that would require several const_casts in the topology tree generation code so it's probably not worth it. --- headers/private/kernel/cpu.h | 2 +- src/system/kernel/cpu.cpp | 2 +- src/system/kernel/int.cpp | 2 +- src/system/kernel/scheduler/scheduler.cpp | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 7b4d51f81a..73614ab424 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -105,7 +105,7 @@ cpu_ent *get_cpu_struct(void); extern inline cpu_ent *get_cpu_struct(void) { return &gCPU[smp_get_current_cpu()]; } status_t cpu_build_topology_tree(void); -cpu_topology_node* get_cpu_topology(void); +const cpu_topology_node* get_cpu_topology(void); status_t increase_cpu_performance(int delta, bool allowBoost); status_t decrease_cpu_performance(int delta); diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 3a3df3a043..2adc443bc8 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -274,7 +274,7 @@ cpu_build_topology_tree(void) } -cpu_topology_node* +const cpu_topology_node* get_cpu_topology(void) { return &sCPUTopology; diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 9fd6d2e884..6601f3ace2 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -391,7 +391,7 @@ restore_interrupts(cpu_status status) static uint32 assign_cpu(void) { - cpu_topology_node* node; + const cpu_topology_node* node; do { int32 nextID = atomic_add(&sLastCPU, 1); node = get_cpu_topology(); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 18c878470e..4801cde8ec 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -751,7 +751,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) static void -traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) +traverse_topology_tree(const cpu_topology_node* node, int packageID, int coreID) { switch (node->level) { case CPU_TOPOLOGY_SMT: @@ -805,7 +805,7 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) } } - cpu_topology_node* root = get_cpu_topology(); + const cpu_topology_node* root = get_cpu_topology(); traverse_topology_tree(root, 0, 0); cpuToCoreDeleter.Detach(); From 1b06228f136128b8094c8a7d954815e29775cfe4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 17 Dec 2013 23:26:37 +0100 Subject: [PATCH 203/273] kernel: Propagate scheduler modes to cpu{freq, idle} modules --- headers/os/drivers/cpufreq.h | 8 +++- headers/os/drivers/cpuidle.h | 8 +++- headers/private/kernel/cpu.h | 6 ++- headers/private/kernel/kscheduler.h | 6 --- .../cpufreq/intel_pstates/intel_pstates.cpp | 25 ++++++++--- .../cpuidle/intel_cstates/intel_cstates.cpp | 41 ++++++++++++++----- src/system/kernel/cpu.cpp | 22 +++++++--- src/system/kernel/scheduler/scheduler.cpp | 6 ++- src/system/kernel/scheduler/scheduler_cpu.cpp | 4 +- 9 files changed, 89 insertions(+), 37 deletions(-) diff --git a/headers/os/drivers/cpufreq.h b/headers/os/drivers/cpufreq.h index ad36f9998a..3eb04b72b5 100644 --- a/headers/os/drivers/cpufreq.h +++ b/headers/os/drivers/cpufreq.h @@ -8,6 +8,8 @@ #include +#include + #define CPUFREQ_MODULES_PREFIX "power/cpufreq" @@ -19,8 +21,10 @@ typedef struct cpufreq_module_info { float rank; - status_t (*increase_performance)(int delta, bool allowBoost); - status_t (*decrease_performance)(int delta); + void (*cpufreq_set_scheduler_mode)(enum scheduler_mode mode); + + status_t (*cpufreq_increase_performance)(int delta); + status_t (*cpufreq_decrease_performance)(int delta); } cpufreq_module_info; diff --git a/headers/os/drivers/cpuidle.h b/headers/os/drivers/cpuidle.h index c22b3e6cc6..2373978af9 100644 --- a/headers/os/drivers/cpuidle.h +++ b/headers/os/drivers/cpuidle.h @@ -8,6 +8,8 @@ #include +#include + #define CPUIDLE_MODULES_PREFIX "power/cpuidle" @@ -17,8 +19,10 @@ typedef struct cpuidle_module_info { float rank; - void (*idle)(void); - void (*wait)(int32* variable, int32 test); + void (*cpuidle_set_scheduler_mode)(enum scheduler_mode mode); + + void (*cpuidle_idle)(void); + void (*cpuidle_wait)(int32* variable, int32 test); } cpuidle_module_info; diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 73614ab424..17b056afdb 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -16,6 +16,8 @@ #include #include +#include + struct kernel_args; @@ -107,7 +109,9 @@ extern inline cpu_ent *get_cpu_struct(void) { return &gCPU[smp_get_current_cpu() status_t cpu_build_topology_tree(void); const cpu_topology_node* get_cpu_topology(void); -status_t increase_cpu_performance(int delta, bool allowBoost); +void cpu_set_scheduler_mode(enum scheduler_mode mode); + +status_t increase_cpu_performance(int delta); status_t decrease_cpu_performance(int delta); void cpu_idle(void); diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index 16316e87c1..bfc6e0203d 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -18,12 +18,6 @@ struct scheduling_analysis; struct SchedulerListener; -enum scheduler_mode { - SCHEDULER_MODE_LOW_LATENCY, - SCHEDULER_MODE_POWER_SAVING, -}; - - #ifdef __cplusplus extern "C" { #endif diff --git a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp index f7f7fe1f9a..5b545fcd1c 100644 --- a/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp +++ b/src/add-ons/kernel/power/cpufreq/intel_pstates/intel_pstates.cpp @@ -25,6 +25,8 @@ static uint16 sMinPState; static uint16 sMaxPState; static uint16 sBoostPState; +static bool sAvoidBoost; + struct CPUEntry { CPUEntry(); @@ -32,7 +34,7 @@ struct CPUEntry { uint16 fCurrentPState; bigtime_t fLastUpdate; -}; +} CACHE_LINE_ALIGN; static CPUEntry* sCPUEntries; @@ -44,6 +46,13 @@ CPUEntry::CPUEntry() } +static void +pstates_set_scheduler_mode(scheduler_mode mode) +{ + sAvoidBoost = mode == SCHEDULER_MODE_POWER_SAVING; +} + + static int measure_pstate(CPUEntry* entry) { @@ -83,7 +92,7 @@ set_pstate(uint16 pstate) static status_t -increase_performance(int delta, bool allowBoost) +pstates_increase_performance(int delta) { CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()]; @@ -93,7 +102,7 @@ increase_performance(int delta, bool allowBoost) int pState = measure_pstate(entry); pState += (sBoostPState - pState) * delta / kCPUPerformanceScaleMax; - if (!allowBoost) + if (sAvoidBoost && pState < (sMaxPState + sBoostPState) / 2) pState = min_c(pState, sMaxPState); set_pstate(pState); @@ -102,7 +111,7 @@ increase_performance(int delta, bool allowBoost) static status_t -decrease_performance(int delta) +pstates_decrease_performance(int delta) { CPUEntry* entry = &sCPUEntries[smp_get_current_cpu()]; @@ -184,6 +193,8 @@ init_pstates() if (sCPUEntries == NULL) return B_NO_MEMORY; + pstates_set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY); + call_all_cpus_sync(set_normal_pstate, NULL); return B_OK; } @@ -224,8 +235,10 @@ static cpufreq_module_info sIntelPStates = { 1.0f, - increase_performance, - decrease_performance, + pstates_set_scheduler_mode, + + pstates_increase_performance, + pstates_decrease_performance, }; diff --git a/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp index d401b451a6..c90b0107e1 100644 --- a/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp +++ b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp @@ -27,17 +27,19 @@ #define INTEL_CSTATES_MODULE_NAME CPUIDLE_MODULES_PREFIX "/intel_cstates/v1" -#define TIME_STEP 250 +#define BASE_TIME_STEP 500 struct CState { uint32 fCode; int fSubStatesCount; - int fSubStatesStep; }; static CState sCStates[CPUIDLE_CSTATE_MAX]; static int sCStateCount; +static int sTimeStep = BASE_TIME_STEP; +static bool sEnableWait = false; + static bigtime_t* sIdleTime; @@ -56,18 +58,31 @@ x86_mwait(uint32 eax, uint32 ecx) static void -idle(void) +cstates_set_scheduler_mode(scheduler_mode mode) +{ + if (mode == SCHEDULER_MODE_POWER_SAVING) { + sTimeStep = BASE_TIME_STEP / 4; + sEnableWait = true; + } else { + sTimeStep = BASE_TIME_STEP; + sEnableWait = false; + } +} + + +static void +cstates_idle(void) { ASSERT(thread_get_current_thread()->pinned_to_cpu > 0); int32 cpu = smp_get_current_cpu(); bigtime_t idleTime = sIdleTime[cpu]; - int state = min_c(idleTime / TIME_STEP, sCStateCount - 1); + int state = min_c(idleTime / sTimeStep, sCStateCount - 1); ASSERT(state >= 0 && state < sCStateCount); - int subState = idleTime % TIME_STEP; - subState /= sCStates[state].fSubStatesStep; + int subState = idleTime % sTimeStep; + subState /= sTimeStep / sCStates[state].fSubStatesCount; ASSERT(subState >= 0 && subState < sCStates[state].fSubStatesCount); @@ -84,8 +99,11 @@ idle(void) static void -wait(int32* variable, int32 test) +cstates_wait(int32* variable, int32 test) { + if (!sEnableWait) + return; + InterruptsLocker _; x86_monitor(variable, 0, 0); if (*variable != test) @@ -133,7 +151,6 @@ init_cstates() sCStates[sCStateCount].fCode = sCStateCount * 0x10; sCStates[sCStateCount].fSubStatesCount = subStates; - sCStates[sCStateCount].fSubStatesStep = TIME_STEP / subStates; sCStateCount++; } @@ -145,6 +162,8 @@ init_cstates() return B_NO_MEMORY; memset(sIdleTime, 0, sizeof(bigtime_t) * smp_get_num_cpus()); + cstates_set_scheduler_mode(SCHEDULER_MODE_LOW_LATENCY); + dprintf("using Intel C-States: C0%s\n", cStates); return B_OK; } @@ -183,8 +202,10 @@ static cpuidle_module_info sIntelCStates = { 0.8f, - idle, - wait + cstates_set_scheduler_mode, + + cstates_idle, + cstates_wait }; diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index 2adc443bc8..a0ffcaf7f4 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -281,11 +281,21 @@ get_cpu_topology(void) } -status_t -increase_cpu_performance(int delta, bool allowBoost) +void +cpu_set_scheduler_mode(enum scheduler_mode mode) { if (sCPUPerformanceModule != NULL) - return sCPUPerformanceModule->increase_performance(delta, allowBoost); + sCPUPerformanceModule->cpufreq_set_scheduler_mode(mode); + if (sCPUIdleModule != NULL) + sCPUIdleModule->cpuidle_set_scheduler_mode(mode); +} + + +status_t +increase_cpu_performance(int delta) +{ + if (sCPUPerformanceModule != NULL) + return sCPUPerformanceModule->cpufreq_increase_performance(delta); return B_NOT_SUPPORTED; } @@ -294,7 +304,7 @@ status_t decrease_cpu_performance(int delta) { if (sCPUPerformanceModule != NULL) - return sCPUPerformanceModule->decrease_performance(delta); + return sCPUPerformanceModule->cpufreq_decrease_performance(delta); return B_NOT_SUPPORTED; } @@ -308,7 +318,7 @@ cpu_idle(void) #endif if (sCPUIdleModule != NULL) - sCPUIdleModule->idle(); + sCPUIdleModule->cpuidle_idle(); else arch_cpu_idle(); } @@ -318,7 +328,7 @@ void cpu_wait(int32* variable, int32 test) { if (sCPUIdleModule != NULL) - sCPUIdleModule->wait(variable, test); + sCPUIdleModule->cpuidle_wait(variable, test); else arch_cpu_pause(); } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 4801cde8ec..5484b45428 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -996,7 +996,11 @@ _user_estimate_max_scheduling_latency(thread_id id) status_t _user_set_scheduler_mode(int32 mode) { - return scheduler_set_operation_mode(static_cast(mode)); + scheduler_mode schedulerMode = static_cast(mode); + status_t error = scheduler_set_operation_mode(schedulerMode); + if (error == B_OK) + cpu_set_scheduler_mode(schedulerMode); + return error; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index c78d62a149..3c8203ac12 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -215,13 +215,11 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) decrease_cpu_performance(delta); } else { - bool allowBoost = !gCurrentMode->avoid_boost; - int32 delta = load - kTargetLoad; delta *= kMaxLoad - kTargetLoad; delta /= kCPUPerformanceScaleMax; - increase_cpu_performance(delta, allowBoost); + increase_cpu_performance(delta); } } From 5a7289d7209b101262421c6263a4efe1236e1d04 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 18 Dec 2013 00:03:56 +0100 Subject: [PATCH 204/273] scheduler: Fix unassigning threads from core being disabled --- src/system/kernel/scheduler/scheduler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 5484b45428..b6d303bb65 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -677,7 +677,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) gIdlePackageList.Remove(package); // get rid of threads - thread_map(unassign_thread, &core->fCoreID); + thread_map(unassign_thread, core); core->fThreadCount = 0; while (core->fRunQueue.PeekMaximum() != NULL) { From e3d001ff02e087a2392c2c46a7ac2d78d3bc12f6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 19 Dec 2013 19:20:10 +0100 Subject: [PATCH 205/273] x86: Implement multicast ICIs --- headers/private/kernel/arch/smp.h | 3 + headers/private/kernel/arch/x86/apic.h | 10 +- headers/private/kernel/arch/x86/arch_cpu.h | 2 + src/system/kernel/arch/x86/apic.cpp | 111 +++++++++++--------- src/system/kernel/arch/x86/arch_smp.cpp | 112 +++++++++++++++------ src/system/kernel/smp.cpp | 12 ++- 6 files changed, 167 insertions(+), 83 deletions(-) diff --git a/headers/private/kernel/arch/smp.h b/headers/private/kernel/arch/smp.h index f85fcf136f..440617bd1a 100644 --- a/headers/private/kernel/arch/smp.h +++ b/headers/private/kernel/arch/smp.h @@ -10,6 +10,8 @@ struct kernel_args; +class CPUSet; + #ifdef __cplusplus extern "C" { @@ -19,6 +21,7 @@ status_t arch_smp_init(struct kernel_args *args); status_t arch_smp_per_cpu_init(struct kernel_args *args, int32 cpu); void arch_smp_send_ici(int32 target_cpu); void arch_smp_send_broadcast_ici(void); +void arch_smp_send_multicast_ici(CPUSet& cpuSet); #ifdef __cplusplus } diff --git a/headers/private/kernel/arch/x86/apic.h b/headers/private/kernel/arch/x86/apic.h index 9c3d92284a..a9dbed7852 100644 --- a/headers/private/kernel/arch/x86/apic.h +++ b/headers/private/kernel/arch/x86/apic.h @@ -52,7 +52,7 @@ #define APIC_TRIGGER_MODE_LEVEL (1 << 15) /* Interrupt Command defines */ -#define APIC_INTR_COMMAND_1_MASK 0xfff3f000 +#define APIC_INTR_COMMAND_1_MASK 0xfff32000 #define APIC_INTR_COMMAND_2_MASK 0x00ffffff #define APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL 0 @@ -110,6 +110,7 @@ #if !_BOOT_MODE bool apic_available(); +bool x2apic_available(); uint32 apic_read(uint32 offset); void apic_write(uint32 offset, uint32 data); uint32 apic_local_id(); @@ -122,10 +123,9 @@ void apic_disable_local_ints(); uint32 apic_spurious_intr_vector(); void apic_set_spurious_intr_vector(uint32 config); -uint32 apic_intr_command_1(); -void apic_set_intr_command_1(uint32 config); -uint32 apic_intr_command_2(); -void apic_set_intr_command_2(uint32 config); + +void apic_set_interrupt_command(uint32 destination, uint32 mode); +bool apic_interrupt_delivered(void); uint32 apic_lvt_timer(); void apic_set_lvt_timer(uint32 config); diff --git a/headers/private/kernel/arch/x86/arch_cpu.h b/headers/private/kernel/arch/x86/arch_cpu.h index 516b5fc52d..f745c50c6d 100644 --- a/headers/private/kernel/arch/x86/arch_cpu.h +++ b/headers/private/kernel/arch/x86/arch_cpu.h @@ -326,6 +326,8 @@ typedef struct arch_cpu_info { int model; int extended_model; + uint32 logical_apic_id; + struct X86PagingStructures* active_paging_structures; size_t dr6; // temporary storage for debug registers (cf. diff --git a/src/system/kernel/arch/x86/apic.cpp b/src/system/kernel/arch/x86/apic.cpp index 47587063a3..c6719cc23d 100644 --- a/src/system/kernel/arch/x86/apic.cpp +++ b/src/system/kernel/arch/x86/apic.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include "timers/apic_timer.h" @@ -30,6 +31,12 @@ apic_available() } +bool x2apic_available() +{ + return sX2APIC; +} + + uint32 apic_read(uint32 offset) { @@ -94,6 +101,16 @@ apic_end_of_interrupt() } +uint32 +apic_logical_apic_id() +{ + if (sX2APIC) + return x86_read_msr(IA32_MSR_APIC_LOGICAL_DEST); + else + return apic_read(APIC_LOGICAL_DEST); +} + + void apic_disable_local_ints() { @@ -128,49 +145,36 @@ apic_set_spurious_intr_vector(uint32 config) } -uint32 -apic_intr_command_1() -{ - if (sX2APIC) - return x86_read_msr(IA32_MSR_APIC_INTR_COMMAND) & 0xffffffff; - else - return apic_read(APIC_INTR_COMMAND_1); -} - - void -apic_set_intr_command_1(uint32 config) +apic_set_interrupt_command(uint32 destination, uint32 mode) { if (sX2APIC) { - uint64 value = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND); - arch_cpu_memory_read_write_barrier(); - x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, - (value & 0xffffffff00000000LL) | config); - } else - apic_write(APIC_INTR_COMMAND_1, config); + uint64 command = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND); + command &= APIC_INTR_COMMAND_1_MASK; + command |= (uint64)destination << 32; + command |= mode; + x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, command); + } else { + uint32 command2 = apic_read(APIC_INTR_COMMAND_2) + & APIC_INTR_COMMAND_2_MASK; + command2 |= destination << 24; + apic_write(APIC_INTR_COMMAND_2, command2); + + uint32 command1 = apic_read(APIC_INTR_COMMAND_1) + & APIC_INTR_COMMAND_1_MASK; + command1 |= mode; + apic_write(APIC_INTR_COMMAND_1, command1); + } } -uint32 -apic_intr_command_2() +bool +apic_interrupt_delivered(void) { if (sX2APIC) - return x86_read_msr(IA32_MSR_APIC_INTR_COMMAND) >> 32; + return true; else - return apic_read(APIC_INTR_COMMAND_2); -} - - -void -apic_set_intr_command_2(uint32 config) -{ - if (sX2APIC) { - uint64 value = x86_read_msr(IA32_MSR_APIC_INTR_COMMAND); - arch_cpu_memory_read_write_barrier(); - x86_write_msr(IA32_MSR_APIC_INTR_COMMAND, - (value & 0xffffffff) | ((uint64)config << 32)); - } else - apic_write(APIC_INTR_COMMAND_2, config); + return (apic_read(APIC_INTR_COMMAND_1) & APIC_DELIVERY_STATUS) == 0; } @@ -264,11 +268,6 @@ apic_init(kernel_args *args) dprintf("found x2apic\n"); #if 0 if (!get_safemode_boolean(B_SAFEMODE_DISABLE_X2APIC, false)) { - uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE); - if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) { - x86_write_msr(IA32_MSR_APIC_BASE, apic_base - | IA32_MSR_APIC_BASE_X2APIC); - } sX2APIC = true; return B_OK; } @@ -276,11 +275,6 @@ apic_init(kernel_args *args) dprintf("x2apic disabled per safemode setting\n"); #else if (get_safemode_boolean(B_SAFEMODE_ENABLE_X2APIC, false)) { - uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE); - if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) { - x86_write_msr(IA32_MSR_APIC_BASE, apic_base - | IA32_MSR_APIC_BASE_X2APIC); - } sX2APIC = true; dprintf("x2apic enabled per safemode setting\n"); @@ -306,8 +300,33 @@ apic_init(kernel_args *args) status_t apic_per_cpu_init(kernel_args *args, int32 cpu) { - dprintf("setting up apic for CPU %" B_PRId32 ": apic id %" B_PRIu32 ", " - "version %" B_PRIu32 "\n", cpu, apic_local_id(), apic_version()); + if (sX2APIC) { + uint64 apic_base = x86_read_msr(IA32_MSR_APIC_BASE); + if ((apic_base & IA32_MSR_APIC_BASE_X2APIC) == 0) { + x86_write_msr(IA32_MSR_APIC_BASE, apic_base + | IA32_MSR_APIC_BASE_X2APIC); + } + } + + dprintf("setting up %sapic for CPU %" B_PRId32 ": apic id %" B_PRIu32 ", " + "version %" B_PRIu32 "\n", sX2APIC ? "x2" : "", cpu, apic_local_id(), + apic_version()); + + if (!sX2APIC && cpu < 8) { + apic_write(APIC_DEST_FORMAT, uint32(-1)); + + uint8 logical_apic_id = 1 << cpu; + uint32 value = apic_read(APIC_LOGICAL_DEST); + value &= 0xffffff; + apic_write(APIC_LOGICAL_DEST, value | (logical_apic_id << 24)); + } + + // get logical APIC ID + gCPU[cpu].arch.logical_apic_id = apic_logical_apic_id(); + if (!sX2APIC) + gCPU[cpu].arch.logical_apic_id >>= 24; + dprintf("CPU %" B_PRId32 ": logical apic id: %#" B_PRIx32 "\n", cpu, + gCPU[cpu].arch.logical_apic_id); /* set spurious interrupt vector to 0xff */ uint32 config = apic_spurious_intr_vector() & 0xffffff00; diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index adf89a247c..bb95501f38 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -1,4 +1,5 @@ /* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2002-2005, Axel Dörfler, axeld@pinc-software.de. All rights reserved. * Distributed under the terms of the MIT License. * @@ -25,6 +26,8 @@ #include #include +#include + //#define TRACE_ARCH_SMP #ifdef TRACE_ARCH_SMP @@ -34,6 +37,9 @@ #endif +#define ICI_VECTOR 0xfd + + static uint32 sCPUAPICIds[SMP_MAX_CPUS]; static uint32 sAPICVersions[SMP_MAX_CPUS]; @@ -124,47 +130,95 @@ arch_smp_per_cpu_init(kernel_args *args, int32 cpu) } +void +arch_smp_send_multicast_ici(CPUSet& cpuSet) +{ +#if KDEBUG + if (are_interrupts_enabled()) + panic("arch_smp_send_multicast_ici: called with interrupts enabled"); +#endif + + arch_cpu_memory_write_barrier(); + + int32 i = 0; + int32 cpuCount = smp_get_num_cpus(); + + int32 logicalModeCPUs; + if (x2apic_available()) + logicalModeCPUs = cpuCount; + else + logicalModeCPUs = std::min(cpuCount, int32(8)); + + uint32 destination = 0; + for (; i < logicalModeCPUs; i++) { + if (cpuSet.GetBit(i) && i != smp_get_current_cpu()) + destination |= gCPU[i].arch.logical_apic_id; + } + + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_LOGICAL + | APIC_INTR_COMMAND_1_DEST_FIELD; + + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(destination, mode); + + for (; i < cpuCount; i++) { + if (cpuSet.GetBit(i)) { + uint32 destination = sCPUAPICIds[i]; + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL + | APIC_INTR_COMMAND_1_DEST_FIELD; + + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(destination, mode); + } + } +} + + void arch_smp_send_broadcast_ici(void) { - uint32 config; - cpu_status state = disable_interrupts(); +#if KDEBUG + if (are_interrupts_enabled()) + panic("arch_smp_send_broadcast_ici: called with interrupts enabled"); +#endif - config = apic_intr_command_1() & APIC_INTR_COMMAND_1_MASK; - apic_set_intr_command_1(config | 0xfd | APIC_DELIVERY_MODE_FIXED - | APIC_INTR_COMMAND_1_ASSERT - | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL - | APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF); + arch_cpu_memory_write_barrier(); - restore_interrupts(state); + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL + | APIC_INTR_COMMAND_1_DEST_ALL_BUT_SELF; + + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(0, mode); } void arch_smp_send_ici(int32 target_cpu) { - uint32 config; - uint32 timeout; - cpu_status state; +#if KDEBUG + if (are_interrupts_enabled()) + panic("arch_smp_send_ici: called with interrupts enabled"); +#endif - state = disable_interrupts(); + arch_cpu_memory_write_barrier(); - config = apic_intr_command_2() & APIC_INTR_COMMAND_2_MASK; - apic_set_intr_command_2(config | sCPUAPICIds[target_cpu] << 24); + uint32 destination = sCPUAPICIds[target_cpu]; + uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED + | APIC_INTR_COMMAND_1_ASSERT + | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL + | APIC_INTR_COMMAND_1_DEST_FIELD; - config = apic_intr_command_1() & APIC_INTR_COMMAND_1_MASK; - apic_set_intr_command_1(config | 0xfd | APIC_DELIVERY_MODE_FIXED - | APIC_INTR_COMMAND_1_ASSERT - | APIC_INTR_COMMAND_1_DEST_MODE_PHYSICAL - | APIC_INTR_COMMAND_1_DEST_FIELD); - - timeout = 100000000; - // wait for message to be sent - while ((apic_intr_command_1() & APIC_DELIVERY_STATUS) != 0 && --timeout != 0) - asm volatile ("pause;"); - - if (timeout == 0) - panic("arch_smp_send_ici: timeout, target_cpu %" B_PRId32, target_cpu); - - restore_interrupts(state); + while (!apic_interrupt_delivered()) + cpu_pause(); + apic_set_interrupt_command(destination, mode); } + diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index bc7f58323b..780912abc8 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -1112,11 +1112,16 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, if (!sICIEnabled) return; + int currentCPU = smp_get_current_cpu(); + bool broadcast = true; + // count target CPUs int32 targetCPUs = 0; for (int32 i = 0; i < sNumCPUs; i++) { if (cpuMask.GetBit(i)) targetCPUs++; + else if (i != currentCPU) + broadcast = false; } // find_free_message leaves interrupts disabled @@ -1132,7 +1137,6 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, msg->flags = flags; msg->done = 0; - int currentCPU = smp_get_current_cpu(); msg->proc_bitmap = cpuMask; msg->proc_bitmap.ClearBit(currentCPU); if (msg->proc_bitmap.IsEmpty()) { @@ -1153,8 +1157,10 @@ smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, atomic_add(&gCPU[i].ici_counter, 1); } - arch_smp_send_broadcast_ici(); - // TODO: Introduce a call that only bothers the target CPUs! + if (broadcast) + arch_smp_send_broadcast_ici(); + else + arch_smp_send_multicast_ici(cpuMask); if ((flags & SMP_MSG_FLAG_SYNC) != 0) { // wait for the other cpus to finish processing it From 3add52e8babe6ab241987c757cd5dc7b1b53c764 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 19 Dec 2013 20:24:34 +0100 Subject: [PATCH 206/273] kernel: Fix topology node count being decreased twice per node --- src/system/kernel/system_info.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index 82929d0c4b..7f7287c232 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -574,7 +574,7 @@ generate_topology_array(cpu_topology_node_info* topology, count--; topology++; - for (int32 i = 0; i < node->children_count && count > 0; i++, count--) + for (int32 i = 0; i < node->children_count && count > 0; i++) topology = generate_topology_array(topology, node->children[i], count); return topology; } From caf1b0dffbf9326f2f5f542d1cdaf831df86cbb9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 19 Dec 2013 20:47:22 +0100 Subject: [PATCH 207/273] intel_cstates: Computing C-state requires constant time step --- .../kernel/power/cpuidle/intel_cstates/intel_cstates.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp index c90b0107e1..9827c916fd 100644 --- a/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp +++ b/src/add-ons/kernel/power/cpuidle/intel_cstates/intel_cstates.cpp @@ -76,13 +76,14 @@ cstates_idle(void) ASSERT(thread_get_current_thread()->pinned_to_cpu > 0); int32 cpu = smp_get_current_cpu(); + bigtime_t timeStep = sTimeStep; bigtime_t idleTime = sIdleTime[cpu]; - int state = min_c(idleTime / sTimeStep, sCStateCount - 1); + int state = min_c(idleTime / timeStep, sCStateCount - 1); ASSERT(state >= 0 && state < sCStateCount); - int subState = idleTime % sTimeStep; - subState /= sTimeStep / sCStates[state].fSubStatesCount; + int subState = idleTime % timeStep; + subState /= timeStep / sCStates[state].fSubStatesCount; ASSERT(subState >= 0 && subState < sCStates[state].fSubStatesCount); From 663951443784bb63d60abe742f1d1379fb153e18 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 01:07:08 +0100 Subject: [PATCH 208/273] x86[_64]: Support assigning MSI IRQs to arbitrary CPU --- headers/private/kernel/arch/x86/arch_int.h | 9 ++ headers/private/kernel/arch/x86/msi.h | 1 + headers/private/kernel/int.h | 11 +- src/system/kernel/arch/x86/arch_int.cpp | 26 ++++- src/system/kernel/arch/x86/ioapic.cpp | 8 ++ src/system/kernel/arch/x86/msi.cpp | 27 ++++- src/system/kernel/int.cpp | 125 +++++++++++++-------- 7 files changed, 152 insertions(+), 55 deletions(-) diff --git a/headers/private/kernel/arch/x86/arch_int.h b/headers/private/kernel/arch/x86/arch_int.h index 5de0dcb9d1..bfbe401061 100644 --- a/headers/private/kernel/arch/x86/arch_int.h +++ b/headers/private/kernel/arch/x86/arch_int.h @@ -11,6 +11,13 @@ #define NUM_IO_VECTORS (256 - ARCH_INTERRUPT_BASE) +enum irq_source { + IRQ_SOURCE_INVALID, + IRQ_SOURCE_IOAPIC, + IRQ_SOURCE_MSI, +}; + + static inline void arch_int_enable_interrupts_inline(void) { @@ -72,6 +79,8 @@ typedef struct interrupt_controller_s { } interrupt_controller; +void x86_set_irq_source(int irq, irq_source source); + void arch_int_set_interrupt_controller(const interrupt_controller &controller); #endif // __cplusplus diff --git a/headers/private/kernel/arch/x86/msi.h b/headers/private/kernel/arch/x86/msi.h index 18685cb021..04e0cd9674 100644 --- a/headers/private/kernel/arch/x86/msi.h +++ b/headers/private/kernel/arch/x86/msi.h @@ -28,5 +28,6 @@ bool msi_supported(); status_t msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address, uint16 *data); void msi_free_vectors(uint8 count, uint8 startVector); +void msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu); #endif // _KERNEL_ARCH_x86_MSI_H diff --git a/headers/private/kernel/int.h b/headers/private/kernel/int.h index 6922aa0573..91608a4782 100644 --- a/headers/private/kernel/int.h +++ b/headers/private/kernel/int.h @@ -32,13 +32,13 @@ enum interrupt_type { struct irq_assignment { list_link link; + uint32 irq; + uint32 count; + + int32 handlers_count; - spinlock load_lock; - bigtime_t last_measure_time; - bigtime_t last_measure_active; int32 load; - int32 cpu; }; @@ -79,7 +79,8 @@ are_interrupts_enabled(void) status_t reserve_io_interrupt_vectors(long count, long startVector, enum interrupt_type type); -status_t allocate_io_interrupt_vectors(long count, long *startVector); +status_t allocate_io_interrupt_vectors(long count, long *startVector, + enum interrupt_type type); void free_io_interrupt_vectors(long count, long startVector); void assign_io_interrupt_to_cpu(long vector, int32 cpu); diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 41176d850f..0ff467a37c 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include @@ -41,6 +42,8 @@ #endif +static irq_source sVectorSources[NUM_IO_VECTORS]; + static const char *kInterruptNames[] = { /* 0 */ "Divide Error Exception", /* 1 */ "Debug Exception", @@ -331,6 +334,13 @@ x86_page_fault_exception(struct iframe* frame) } +void +x86_set_irq_source(int irq, irq_source source) +{ + sVectorSources[irq] = source; +} + + // #pragma mark - @@ -392,8 +402,20 @@ arch_int_are_interrupts_enabled(void) void arch_int_assign_to_cpu(int32 irq, int32 cpu) { - if (sCurrentPIC->assign_interrupt_to_cpu != NULL) - sCurrentPIC->assign_interrupt_to_cpu(irq, cpu); + dprintf("ASSIGN IRQ TO CPU\n"); + switch (sVectorSources[irq]) { + case IRQ_SOURCE_IOAPIC: + if (sCurrentPIC->assign_interrupt_to_cpu != NULL) + sCurrentPIC->assign_interrupt_to_cpu(irq, cpu); + break; + + case IRQ_SOURCE_MSI: + msi_assign_interrupt_to_cpu(irq, cpu); + break; + + default: + break; + } } diff --git a/src/system/kernel/arch/x86/ioapic.cpp b/src/system/kernel/arch/x86/ioapic.cpp index 3f6c5448f6..bbc8d72ff9 100644 --- a/src/system/kernel/arch/x86/ioapic.cpp +++ b/src/system/kernel/arch/x86/ioapic.cpp @@ -280,6 +280,8 @@ ioapic_enable_io_interrupt(int32 gsi) if (ioapic == NULL) return; + x86_set_irq_source(gsi, IRQ_SOURCE_IOAPIC); + uint8 pin = gsi - ioapic->global_interrupt_base; TRACE("ioapic_enable_io_interrupt: gsi %ld -> io-apic %u pin %u\n", gsi, ioapic->number, pin); @@ -801,6 +803,12 @@ ioapic_init(kernel_args* args) while (current != NULL) { reserve_io_interrupt_vectors(current->max_redirection_entry + 1, current->global_interrupt_base, INTERRUPT_TYPE_IRQ); + + for (int32 i = 0; i < current->max_redirection_entry + 1; i++) { + x86_set_irq_source(current->global_interrupt_base + i, + IRQ_SOURCE_IOAPIC); + } + current = current->next; } diff --git a/src/system/kernel/arch/x86/msi.cpp b/src/system/kernel/arch/x86/msi.cpp index 54f4cffc28..4578e14d19 100644 --- a/src/system/kernel/arch/x86/msi.cpp +++ b/src/system/kernel/arch/x86/msi.cpp @@ -5,12 +5,20 @@ #include #include +#include #include #include #include +struct MSIConfiguration { + uint64* fAddress; + uint16* fData; +}; + +static MSIConfiguration sMSIConfigurations[NUM_IO_VECTORS]; + static bool sMSISupported = false; static uint32 sBootCPUAPICId = 0; @@ -44,7 +52,8 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address, return B_UNSUPPORTED; long vector; - status_t result = allocate_io_interrupt_vectors(count, &vector); + status_t result = allocate_io_interrupt_vectors(count, &vector, + INTERRUPT_TYPE_IRQ); if (result != B_OK) return result; @@ -53,6 +62,10 @@ msi_allocate_vectors(uint8 count, uint8 *startVector, uint64 *address, return B_NO_MEMORY; } + sMSIConfigurations[vector].fAddress = address; + sMSIConfigurations[vector].fData = data; + x86_set_irq_source(vector, IRQ_SOURCE_MSI); + *startVector = (uint8)vector; *address = MSI_ADDRESS_BASE | (sBootCPUAPICId << MSI_DESTINATION_ID_SHIFT) | MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL; @@ -78,3 +91,15 @@ msi_free_vectors(uint8 count, uint8 startVector) free_io_interrupt_vectors(count, startVector); } + + +void +msi_assign_interrupt_to_cpu(uint8 irq, int32 cpu) +{ + uint32 apic_id = x86_get_cpu_apic_id(cpu); + + uint64* address = sMSIConfigurations[irq].fAddress; + *address = MSI_ADDRESS_BASE | (apic_id << MSI_DESTINATION_ID_SHIFT) + | MSI_NO_REDIRECTION | MSI_DESTINATION_MODE_PHYSICAL; +} + diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index 6601f3ace2..fea008b845 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -57,7 +57,12 @@ struct io_vector { bool no_lock_vector; interrupt_type type; - irq_assignment assigned_cpu; + spinlock load_lock; + bigtime_t last_measure_time; + bigtime_t last_measure_active; + int32 load; + + irq_assignment* assigned_cpu; #if DEBUG_INTERRUPTS int64 handled_count; @@ -69,8 +74,9 @@ struct io_vector { static int32 sLastCPU; -static struct io_vector sVectors[NUM_IO_VECTORS]; +static io_vector sVectors[NUM_IO_VECTORS]; static bool sAllocatedIOInterruptVectors[NUM_IO_VECTORS]; +static irq_assignment sVectorCPUAssignments[NUM_IO_VECTORS]; static mutex sIOInterruptVectorAllocationLock = MUTEX_INITIALIZER("io_interrupt_vector_allocation"); @@ -141,11 +147,15 @@ dump_int_load(int argc, char** argv) kprintf("int %3d, type %s, enabled %" B_PRId32 ", load %" B_PRId32 "%%", i, typeNames[min_c(sVectors[i].type, INTERRUPT_TYPE_UNKNOWN)], - sVectors[i].enable_count, sVectors[i].assigned_cpu.load / 10); + sVectors[i].enable_count, + sVectors[i].assigned_cpu != NULL + ? sVectors[i].assigned_cpu->load / 10 : 0); if (sVectors[i].type == INTERRUPT_TYPE_IRQ) { - if (sVectors[i].assigned_cpu.cpu != -1) - kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu.cpu); + ASSERT(sVectors[i].assigned_cpu != NULL); + + if (sVectors[i].assigned_cpu->cpu != -1) + kprintf(", cpu %" B_PRId32, sVectors[i].assigned_cpu->cpu); else kprintf(", cpu -"); } @@ -190,15 +200,10 @@ int_init_post_vm(kernel_args* args) sVectors[i].no_lock_vector = false; sVectors[i].type = INTERRUPT_TYPE_UNKNOWN; - irq_assignment* assigned_cpu = &sVectors[i].assigned_cpu; - assigned_cpu->irq = i; - - B_INITIALIZE_SPINLOCK(&assigned_cpu->load_lock); - assigned_cpu->last_measure_time = 0; - assigned_cpu->last_measure_active = 0; - assigned_cpu->load = 0; - - assigned_cpu->cpu = -1; + B_INITIALIZE_SPINLOCK(&sVectors[i].load_lock); + sVectors[i].last_measure_time = 0; + sVectors[i].last_measure_active = 0; + sVectors[i].load = 0; #if DEBUG_INTERRUPTS sVectors[i].handled_count = 0; @@ -207,6 +212,12 @@ int_init_post_vm(kernel_args* args) sVectors[i].ignored_count = 0; #endif sVectors[i].handler_list = NULL; + + sVectorCPUAssignments[i].irq = i; + sVectorCPUAssignments[i].count = 1; + sVectorCPUAssignments[i].handlers_count = 0; + sVectorCPUAssignments[i].load = 0; + sVectorCPUAssignments[i].cpu = -1; } #if DEBUG_INTERRUPTS @@ -240,14 +251,17 @@ int_init_post_device_manager(kernel_args* args) static void update_int_load(int i) { - if (!try_acquire_spinlock(&sVectors[i].assigned_cpu.load_lock)) + if (!try_acquire_spinlock(&sVectors[i].load_lock)) return; - compute_load(sVectors[i].assigned_cpu.last_measure_time, - sVectors[i].assigned_cpu.last_measure_active, - sVectors[i].assigned_cpu.load); + int32 oldLoad = sVectors[i].load; + compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active, + sVectors[i].load); - release_spinlock(&sVectors[i].assigned_cpu.load_lock); + if (oldLoad != sVectors[i].load) + atomic_add(&sVectors[i].assigned_cpu->load, sVectors[i].load - oldLoad); + + release_spinlock(&sVectors[i].load_lock); } @@ -345,9 +359,9 @@ int_io_interrupt_handler(int vector, bool levelTriggered) if (!sVectors[vector].no_lock_vector) release_spinlock(&sVectors[vector].vector_lock); - SpinLocker locker(sVectors[vector].assigned_cpu.load_lock); + SpinLocker locker(sVectors[vector].load_lock); bigtime_t deltaTime = system_time() - start; - sVectors[vector].assigned_cpu.last_measure_active += deltaTime; + sVectors[vector].last_measure_active += deltaTime; locker.Unlock(); atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime); @@ -444,18 +458,17 @@ install_io_interrupt_handler(long vector, interrupt_handler handler, void *data, // Initial attempt to balance IRQs, the scheduler will correct this // if some cores end up being overloaded. if (sVectors[vector].type == INTERRUPT_TYPE_IRQ - && sVectors[vector].handler_list == NULL) { + && sVectors[vector].handler_list == NULL + && sVectors[vector].assigned_cpu->cpu == -1) { int32 cpuID = assign_cpu(); arch_int_assign_to_cpu(vector, cpuID); - - ASSERT(sVectors[vector].assigned_cpu.cpu == -1); - - sVectors[vector].assigned_cpu.cpu = cpuID; + sVectors[vector].assigned_cpu->cpu = cpuID; cpu_ent* cpu = &gCPU[cpuID]; SpinLocker _(cpu->irqs_lock); - list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + atomic_add(&sVectors[vector].assigned_cpu->handlers_count, 1); + list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu); } if ((flags & B_NO_HANDLED_INFO) != 0 @@ -544,25 +557,32 @@ remove_io_interrupt_handler(long vector, interrupt_handler handler, void *data) } if (sVectors[vector].handler_list == NULL - && sVectors[vector].type == INTERRUPT_TYPE_IRQ) { + && sVectors[vector].type == INTERRUPT_TYPE_IRQ + && sVectors[vector].assigned_cpu != NULL + && sVectors[vector].assigned_cpu->handlers_count > 0) { - int32 oldCPU; - SpinLocker locker; - cpu_ent* cpu; + int32 oldHandlersCount + = atomic_add(&sVectors[vector].assigned_cpu->handlers_count, -1); - do { - locker.Unlock(); + if (oldHandlersCount == 1) { + int32 oldCPU; + SpinLocker locker; + cpu_ent* cpu; - oldCPU = sVectors[vector].assigned_cpu.cpu; + do { + locker.Unlock(); - ASSERT(oldCPU != -1); - cpu = &gCPU[oldCPU]; + oldCPU = sVectors[vector].assigned_cpu->cpu; - locker.SetTo(cpu->irqs_lock, false); - } while (sVectors[vector].assigned_cpu.cpu != oldCPU); + ASSERT(oldCPU != -1); + cpu = &gCPU[oldCPU]; - sVectors[vector].assigned_cpu.cpu = -1; - list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + locker.SetTo(cpu->irqs_lock, false); + } while (sVectors[vector].assigned_cpu->cpu != oldCPU); + + sVectors[vector].assigned_cpu->cpu = -1; + list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu); + } } release_spinlock(&sVectors[vector].vector_lock); @@ -596,6 +616,9 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type) } sVectors[startVector + i].type = type; + sVectors[startVector + i].assigned_cpu + = &sVectorCPUAssignments[startVector + i]; + sVectorCPUAssignments[startVector + i].count = 1; sAllocatedIOInterruptVectors[startVector + i] = true; } @@ -610,7 +633,8 @@ reserve_io_interrupt_vectors(long count, long startVector, interrupt_type type) The first vector to be used is returned in \a startVector on success. */ status_t -allocate_io_interrupt_vectors(long count, long *startVector) +allocate_io_interrupt_vectors(long count, long *startVector, + interrupt_type type) { MutexLocker locker(&sIOInterruptVectorAllocationLock); @@ -639,8 +663,14 @@ allocate_io_interrupt_vectors(long count, long *startVector) return B_NO_MEMORY; } - for (long i = 0; i < count; i++) + for (long i = 0; i < count; i++) { + sVectors[vector + i].type = type; + sVectors[vector + i].assigned_cpu = &sVectorCPUAssignments[vector]; sAllocatedIOInterruptVectors[vector + i] = true; + } + + sVectorCPUAssignments[vector].irq = vector; + sVectorCPUAssignments[vector].count = count; *startVector = vector; dprintf("allocate_io_interrupt_vectors: allocated %ld vectors starting " @@ -672,6 +702,7 @@ free_io_interrupt_vectors(long count, long startVector) startVector + i); } + sVectors[startVector + i].assigned_cpu = NULL; sAllocatedIOInterruptVectors[startVector + i] = false; } } @@ -681,7 +712,7 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) { ASSERT(sVectors[vector].type == INTERRUPT_TYPE_IRQ); - int32 oldCPU = sVectors[vector].assigned_cpu.cpu; + int32 oldCPU = sVectors[vector].assigned_cpu->cpu; if (newCPU == -1) newCPU = assign_cpu(); @@ -693,14 +724,14 @@ void assign_io_interrupt_to_cpu(long vector, int32 newCPU) cpu_ent* cpu = &gCPU[oldCPU]; SpinLocker locker(cpu->irqs_lock); - sVectors[vector].assigned_cpu.cpu = -1; - list_remove_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + sVectors[vector].assigned_cpu->cpu = -1; + list_remove_item(&cpu->irqs, sVectors[vector].assigned_cpu); locker.Unlock(); cpu = &gCPU[newCPU]; locker.SetTo(cpu->irqs_lock, false); - sVectors[vector].assigned_cpu.cpu = newCPU; + sVectors[vector].assigned_cpu->cpu = newCPU; arch_int_assign_to_cpu(vector, newCPU); - list_add_item(&cpu->irqs, &sVectors[vector].assigned_cpu); + list_add_item(&cpu->irqs, sVectors[vector].assigned_cpu); } From b258298c70249e60ea7c65c60bd5ee1250609921 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 01:31:32 +0100 Subject: [PATCH 209/273] kernel: Protect cpu_ent::active_time with sequential lock atomic_{get, set}64() are problematic on architectures without 64 bit compare and swap. Also, using sequential lock instead of atomic access ensures that any reads from cpu_ent::active_time won't require any writes to shared memory. --- headers/private/kernel/cpu.h | 1 + src/system/kernel/cpu.cpp | 10 +++++++++- src/system/kernel/scheduler/scheduler_cpu.cpp | 5 ++++- src/system/kernel/smp.cpp | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/headers/private/kernel/cpu.h b/headers/private/kernel/cpu.h index 17b056afdb..dad23c6c56 100644 --- a/headers/private/kernel/cpu.h +++ b/headers/private/kernel/cpu.h @@ -56,6 +56,7 @@ typedef struct cpu_ent { timer quantum_timer; // keeping track of CPU activity + seqlock active_time_lock; bigtime_t active_time; bigtime_t irq_time; bigtime_t interrupt_time; diff --git a/src/system/kernel/cpu.cpp b/src/system/kernel/cpu.cpp index a0ffcaf7f4..c984f353d1 100644 --- a/src/system/kernel/cpu.cpp +++ b/src/system/kernel/cpu.cpp @@ -159,7 +159,15 @@ cpu_get_active_time(int32 cpu) if (cpu < 0 || cpu > smp_get_num_cpus()) return 0; - return atomic_get64(&gCPU[cpu].active_time); + bigtime_t activeTime; + uint32 count; + + do { + count = acquire_read_seqlock(&gCPU[cpu].active_time_lock); + activeTime = gCPU[cpu].active_time; + } while (!release_read_seqlock(&gCPU[cpu].active_time_lock, count)); + + return activeTime; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 3c8203ac12..57544ae83e 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -180,7 +180,10 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) = (oldThread->kernel_time - cpuEntry->last_kernel_time) + (oldThread->user_time - cpuEntry->last_user_time); - atomic_add64(&cpuEntry->active_time, active); + WriteSequentialLocker locker(cpuEntry->active_time_lock); + cpuEntry->active_time += active; + locker.Unlock(); + oldThreadData->UpdateActivity(active); } diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 780912abc8..3631234771 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -701,6 +701,7 @@ acquire_read_seqlock(seqlock* lock) { bool release_read_seqlock(seqlock* lock, uint32 count) { + arch_cpu_memory_read_barrier(); uint32 current = atomic_get((int32*)&lock->count); if (count % 2 == 1 || current != count) { From ad6b9a1df8ccdb1093c4b122764f8692d6f7ca2c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 02:18:44 +0100 Subject: [PATCH 210/273] scheduler: Use sequential locks instead of atomic 64 bit access --- headers/os/drivers/KernelExport.h | 6 +++--- src/system/kernel/scheduler/low_latency.cpp | 12 ++++++++++-- src/system/kernel/scheduler/scheduler_cpu.cpp | 1 + src/system/kernel/scheduler/scheduler_cpu.h | 1 + src/system/kernel/scheduler/scheduler_thread.h | 12 ++++++++++-- 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/headers/os/drivers/KernelExport.h b/headers/os/drivers/KernelExport.h index 4f7e8e2969..ffdbb245ea 100644 --- a/headers/os/drivers/KernelExport.h +++ b/headers/os/drivers/KernelExport.h @@ -59,9 +59,9 @@ typedef struct { } seqlock; #define B_SEQLOCK_INITIALIZER { B_SPINLOCK_INITIALIZER, 0 } -#define B_INITIALIZE_SEQLOCK(seqlock) do { \ - B_INITIALIZE_SPINLOCK((seqlock)->lock); \ - (seqlock)->count = 0; \ +#define B_INITIALIZE_SEQLOCK(seqlock) do { \ + B_INITIALIZE_SPINLOCK(&(seqlock)->lock); \ + (seqlock)->count = 0; \ } while (false) /* interrupt handling support for device drivers */ diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index a272f53214..409699792f 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -35,8 +35,16 @@ has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - return atomic_get64(&threadData->GetCore()->fActiveTime) - - threadData->fWentSleepActive > kCacheExpire; + CoreEntry* core = threadData->GetCore(); + + bigtime_t activeTime; + uint32 count; + do { + count = acquire_read_seqlock(&core->fActiveTimeLock); + activeTime = core->fActiveTime; + } while (!release_read_seqlock(&core->fActiveTimeLock, count)); + + return activeTime - threadData->fWentSleepActive > kCacheExpire; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 57544ae83e..bd812677c2 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -272,6 +272,7 @@ CoreEntry::CoreEntry() { B_INITIALIZE_SPINLOCK(&fCPULock); B_INITIALIZE_SPINLOCK(&fQueueLock); + B_INITIALIZE_SEQLOCK(&fActiveTimeLock); } diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 14c1a80836..b2f5727cff 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -100,6 +100,7 @@ struct CoreEntry : public MinMaxHeapLinkImpl, spinlock fQueueLock; bigtime_t fActiveTime; + seqlock fActiveTimeLock; int32 fLoad; bool fHighLoad; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index ffdae08035..9c98b7e7d0 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -177,8 +177,13 @@ ThreadData::GoesAway() fLastInterruptTime = 0; fWentSleep = system_time(); - fWentSleepActive = atomic_get64(&fCore->fActiveTime); fWentSleepCount = atomic_get(&fCore->fStarvationCounter); + + uint32 count; + do { + count = acquire_read_seqlock(&fCore->fActiveTimeLock); + fWentSleepActive = fCore->fActiveTime; + } while (!release_read_seqlock(&fCore->fActiveTimeLock, count)); } @@ -262,7 +267,10 @@ ThreadData::UpdateActivity(bigtime_t active) { fMeasureActiveTime += active; gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - atomic_add64(&fCore->fActiveTime, active); + + WriteSequentialLocker locker(fCore->fActiveTimeLock); + fCore->fActiveTime += active; + locker.Unlock(); } From c08ed2db65267bea18a3ba424f98fffde9da6c25 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 03:36:01 +0100 Subject: [PATCH 211/273] scheduler: Try to keep thread on the same logical CPU Some SMT implementations (e.g. recent AMD microarchitectures) have separate L1d cache for each SMT thread (which AMD decides to call "cores"). This means that we shouldn't move threads to another logical processor too often even if it belongs to the same core. We aren't very strict about this as it would complicate load balancing, but we try to reduce unnecessary migrations. --- src/system/kernel/scheduler/scheduler.cpp | 11 +++--- src/system/kernel/scheduler/scheduler_cpu.cpp | 1 - src/system/kernel/scheduler/scheduler_cpu.h | 1 - .../kernel/scheduler/scheduler_thread.cpp | 39 +++++++++++++++++++ .../kernel/scheduler/scheduler_thread.h | 26 ------------- 5 files changed, 44 insertions(+), 34 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index b6d303bb65..dcb0583d77 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -175,9 +175,8 @@ enqueue(Thread* thread, bool newOne) thread); int32 heapPriority = CPUPriorityHeap::GetKey(targetCPU); - if (threadPriority > atomic_get(&targetCPU->fPriority) - && (threadPriority > heapPriority - || (threadPriority == heapPriority && rescheduleNeeded))) { + if (threadPriority > heapPriority + || (threadPriority == heapPriority && rescheduleNeeded)) { if (targetCPU->fCPUNumber == smp_get_current_cpu()) gCPU[targetCPU->fCPUNumber].invoke_scheduler = true; @@ -237,7 +236,6 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num]; SpinLocker coreLocker(threadData->GetCore()->fCPULock); - cpu->fPriority = priority; cpu->UpdatePriority(priority); } @@ -475,7 +473,9 @@ reschedule(int32 nextState) } Thread* nextThread = nextThreadData->GetThread(); - atomic_set(&cpu->fPriority, nextThreadData->GetEffectivePriority()); + SpinLocker cpuLocker(core->fCPULock); + cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); + cpuLocker.Unlock(); if (nextThread != oldThread) { if (enqueueOldThread) { @@ -650,7 +650,6 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) if (enabled) core->fCPUCount++; else { - cpu->fPriority = B_IDLE_PRIORITY; cpu->UpdatePriority(B_IDLE_PRIORITY); core->fCPUCount--; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index bd812677c2..d55d03d561 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -42,7 +42,6 @@ ThreadRunQueue::Dump() const CPUEntry::CPUEntry() : - fPriority(B_IDLE_PRIORITY), fLoad(0), fMeasureActiveTime(0), fMeasureTime(0) diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index b2f5727cff..4d02a646dc 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -54,7 +54,6 @@ struct CPUEntry : public MinMaxHeapLinkImpl { rw_spinlock fSchedulerModeLock; - int32 fPriority; ThreadRunQueue fRunQueue; int32 fLoad; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index cf53fcda9f..e1f280e1c7 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -121,6 +121,45 @@ ThreadData::ComputeQuantum() } +inline CoreEntry* +ThreadData::_ChooseCore() const +{ + ASSERT(!gSingleCore); + return gCurrentMode->choose_core(this); +} + + +inline CPUEntry* +ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const +{ + int32 threadPriority = GetEffectivePriority(); + + if (fThread->previous_cpu != NULL) { + CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; + if (previousCPU->fCore == core) { + SpinLocker cpuLocker(core->fCPULock); + if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { + previousCPU->UpdatePriority(threadPriority); + rescheduleNeeded = true; + return previousCPU; + } + } + } + + SpinLocker cpuLocker(core->fCPULock); + CPUEntry* cpu = core->fCPUHeap.PeekMinimum(); + ASSERT(cpu != NULL); + + if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { + cpu->UpdatePriority(threadPriority); + rescheduleNeeded = true; + } else + rescheduleNeeded = false; + + return cpu; +} + + inline bigtime_t ThreadData::_GetBaseQuantum() const { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 9c98b7e7d0..5f4ab2f12f 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -335,32 +335,6 @@ ThreadData::_GetMinimalPriority() const } -inline CoreEntry* -ThreadData::_ChooseCore() const -{ - ASSERT(!gSingleCore); - return gCurrentMode->choose_core(this); -} - - -inline CPUEntry* -ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const -{ - SpinLocker cpuLocker(core->fCPULock); - CPUEntry* cpu = core->fCPUHeap.PeekMinimum(); - ASSERT(cpu != NULL); - - int32 threadPriority = GetEffectivePriority(); - if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { - cpu->UpdatePriority(threadPriority); - rescheduleNeeded = true; - } else - rescheduleNeeded = false; - - return cpu; -} - - } // namespace Scheduler From 73363f8ddec8f3262485fbf84a48dd8412dff4ed Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 05:01:13 +0100 Subject: [PATCH 212/273] kernel/vm: Fix unsigned/signed comparison warning --- src/system/kernel/vm/vm_page.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/vm/vm_page.cpp b/src/system/kernel/vm/vm_page.cpp index 008782773c..4c01f1f837 100644 --- a/src/system/kernel/vm/vm_page.cpp +++ b/src/system/kernel/vm/vm_page.cpp @@ -4077,7 +4077,7 @@ vm_page_get_stats(system_info *info) // active + inactive + unused + wired + modified + cached + free + clear // So taking out the cached (including modified non-temporary), free and // clear ones leaves us with all used pages. - int32 subtractPages = info->cached_pages + sFreePageQueue.Count() + uint32 subtractPages = info->cached_pages + sFreePageQueue.Count() + sClearPageQueue.Count(); info->used_pages = subtractPages > info->max_pages ? 0 : info->max_pages - subtractPages; From b4552fcf9b2700a356c0dc25f49a052adc504650 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 05:02:23 +0100 Subject: [PATCH 213/273] x86: Remove debug message --- src/system/kernel/arch/x86/arch_int.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/system/kernel/arch/x86/arch_int.cpp b/src/system/kernel/arch/x86/arch_int.cpp index 0ff467a37c..89b82575df 100644 --- a/src/system/kernel/arch/x86/arch_int.cpp +++ b/src/system/kernel/arch/x86/arch_int.cpp @@ -402,7 +402,6 @@ arch_int_are_interrupts_enabled(void) void arch_int_assign_to_cpu(int32 irq, int32 cpu) { - dprintf("ASSIGN IRQ TO CPU\n"); switch (sVectorSources[irq]) { case IRQ_SOURCE_IOAPIC: if (sCurrentPIC->assign_interrupt_to_cpu != NULL) From 9116eec24c3a9e8be6ba721487fa29604fb5b944 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 05:38:15 +0100 Subject: [PATCH 214/273] scheduler: Allow calling UpdatePriority() for disabled CPU --- src/system/kernel/scheduler/scheduler.cpp | 11 +++++------ src/system/kernel/scheduler/scheduler_cpu.cpp | 3 +++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index dcb0583d77..8046bd5bd7 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -501,10 +501,9 @@ reschedule(int32 nextState) nextThread->state = B_THREAD_RUNNING; // update CPU heap - if (!gCPU[thisCPU].disabled) { - SpinLocker coreLocker(core->fCPULock); - cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); - } + SpinLocker coreLocker(core->fCPULock); + cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); + coreLocker.Unlock(); // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); @@ -637,8 +636,6 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) InterruptsBigSchedulerLocker _; - gCPU[cpuID].disabled = !enabled; - gCurrentMode->set_cpu_enabled(cpuID, enabled); CPUEntry* cpu = &gCPUEntries[cpuID]; @@ -654,6 +651,8 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) core->fCPUCount--; } + gCPU[cpuID].disabled = !enabled; + if (core->fCPUCount == 0) { // core has been disabled ASSERT(!enabled); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index d55d03d561..8c9b0489c4 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -53,6 +53,9 @@ CPUEntry::CPUEntry() void CPUEntry::UpdatePriority(int32 priority) { + if (gCPU[fCPUNumber].disabled) + return; + int32 corePriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); fCore->fCPUHeap.ModifyKey(this, priority); From b89b5d3826cd16570d24f6fa29d0ccaa2843d0dd Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 20 Dec 2013 22:05:26 +0100 Subject: [PATCH 215/273] x86: Make arch/smp.h a C++ only header --- headers/private/kernel/arch/smp.h | 13 ++++--------- src/system/kernel/arch/x86/timers/x86_apic.cpp | 1 - 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/headers/private/kernel/arch/smp.h b/headers/private/kernel/arch/smp.h index 440617bd1a..4b59e45041 100644 --- a/headers/private/kernel/arch/smp.h +++ b/headers/private/kernel/arch/smp.h @@ -8,23 +8,18 @@ #include + struct kernel_args; class CPUSet; -#ifdef __cplusplus -extern "C" { -#endif +status_t arch_smp_init(kernel_args* args); +status_t arch_smp_per_cpu_init(kernel_args* args, int32 cpu); -status_t arch_smp_init(struct kernel_args *args); -status_t arch_smp_per_cpu_init(struct kernel_args *args, int32 cpu); void arch_smp_send_ici(int32 target_cpu); -void arch_smp_send_broadcast_ici(void); +void arch_smp_send_broadcast_ici(); void arch_smp_send_multicast_ici(CPUSet& cpuSet); -#ifdef __cplusplus -} -#endif #endif /* KERNEL_ARCH_SMP_H */ diff --git a/src/system/kernel/arch/x86/timers/x86_apic.cpp b/src/system/kernel/arch/x86/timers/x86_apic.cpp index 6ac3e5947f..568cd65244 100644 --- a/src/system/kernel/arch/x86/timers/x86_apic.cpp +++ b/src/system/kernel/arch/x86/timers/x86_apic.cpp @@ -14,7 +14,6 @@ #include #include -#include #include "apic_timer.h" From cf21c40b50b50e4b8dfc8a5c33555be60a3f33e4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 00:09:03 +0100 Subject: [PATCH 216/273] x86: Fix style Thanks Axel. --- src/system/kernel/arch/x86/apic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/arch/x86/apic.cpp b/src/system/kernel/arch/x86/apic.cpp index c6719cc23d..176102977d 100644 --- a/src/system/kernel/arch/x86/apic.cpp +++ b/src/system/kernel/arch/x86/apic.cpp @@ -31,7 +31,8 @@ apic_available() } -bool x2apic_available() +bool +x2apic_available() { return sX2APIC; } From 60e198f2cbf2e26b584370c0d32c37cb3dce556c Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 00:56:44 +0100 Subject: [PATCH 217/273] scheduler: Encapsulate PackageEntry fields Apart from the refactoring this commit takes the opportunity and removes unnecessary read locks when choosing a package and a core from idle lists. The data structures are accessed in a thread safe way and it does not really matter whether the obtained data becomes outdated just when we release the lock or during our search for the appropriate package/core. --- src/system/kernel/scheduler/low_latency.cpp | 32 +--- src/system/kernel/scheduler/power_saving.cpp | 30 ++-- src/system/kernel/scheduler/scheduler.cpp | 79 ++++----- .../kernel/scheduler/scheduler_common.h | 2 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 150 ++++++++++++------ src/system/kernel/scheduler/scheduler_cpu.h | 63 +++++++- src/system/kernel/scheduler/scheduler_modes.h | 2 +- .../kernel/scheduler/scheduler_thread.h | 2 +- 8 files changed, 224 insertions(+), 136 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 409699792f..441a609a7d 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -19,7 +19,7 @@ const bigtime_t kCacheExpire = 100000; static void -switch_to_mode(void) +switch_to_mode() { } @@ -35,7 +35,7 @@ has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - CoreEntry* core = threadData->GetCore(); + CoreEntry* core = threadData->Core(); bigtime_t activeTime; uint32 count; @@ -48,39 +48,19 @@ has_cache_expired(const ThreadData* threadData) } -static inline PackageEntry* -get_most_idle_package(void) -{ - PackageEntry* current = &gPackageEntries[0]; - for (int32 i = 1; i < gPackageCount; i++) { - if (gPackageEntries[i].fIdleCoreCount > current->fIdleCoreCount) - current = &gPackageEntries[i]; - } - - if (current->fIdleCoreCount == 0) - return NULL; - - return current; -} - - static CoreEntry* choose_core(const ThreadData* /* threadData */) { - ReadSpinLocker locker(gIdlePackageLock); // wake new package PackageEntry* package = gIdlePackageList.Last(); if (package == NULL) { // wake new core - package = get_most_idle_package(); + package = PackageEntry::GetMostIdlePackage(); } - locker.Unlock(); CoreEntry* core = NULL; - if (package != NULL) { - ReadSpinLocker _(package->fCoreLock); - core = package->fIdleCores.Last(); - } + if (package != NULL) + core = package->GetIdleCore(); if (core == NULL) { ReadSpinLocker coreLocker(gCoreHeapsLock); @@ -98,7 +78,7 @@ choose_core(const ThreadData* /* threadData */) static bool should_rebalance(const ThreadData* threadData) { - int32 coreLoad = threadData->GetCore()->GetLoad(); + int32 coreLoad = threadData->Core()->GetLoad(); // If the thread produces more than 50% of the load, leave it here. In // such situation it is better to move other threads away. diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index c1eeb61609..2ca77db7ec 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -22,7 +22,7 @@ static CoreEntry* sSmallTaskCore; static void -switch_to_mode(void) +switch_to_mode() { sSmallTaskCore = NULL; } @@ -46,7 +46,7 @@ has_cache_expired(const ThreadData* threadData) static CoreEntry* -choose_small_task_core(void) +choose_small_task_core() { ReadSpinLocker locker(gCoreHeapsLock); CoreEntry* core = gCoreLoadHeap.PeekMaximum(); @@ -64,27 +64,15 @@ choose_small_task_core(void) static CoreEntry* -choose_idle_core(void) +choose_idle_core() { - PackageEntry* package = NULL; + PackageEntry* package = PackageEntry::GetLeastIdlePackage(); - for (int32 i = 0; i < gPackageCount; i++) { - PackageEntry* current = &gPackageEntries[i]; - if (current->fIdleCoreCount != 0 && (package == NULL - || current->fIdleCoreCount < package->fIdleCoreCount)) { - package = current; - } - } - - if (package == NULL) { - ReadSpinLocker _(gIdlePackageLock); + if (package == NULL) package = gIdlePackageList.Last(); - } - if (package != NULL) { - ReadSpinLocker _(package->fCoreLock); - return package->fIdleCores.Last(); - } + if (package != NULL) + return package->GetIdleCore(); return NULL; } @@ -125,7 +113,7 @@ should_rebalance(const ThreadData* threadData) { ASSERT(!gSingleCore); - CoreEntry* core = threadData->GetCore(); + CoreEntry* core = threadData->Core(); int32 coreLoad = core->GetLoad(); if (coreLoad > kHighLoad) { @@ -161,7 +149,7 @@ should_rebalance(const ThreadData* threadData) static inline void -pack_irqs(void) +pack_irqs() { CoreEntry* smallTaskCore = atomic_pointer_get(&sSmallTaskCore); if (smallTaskCore == NULL) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 8046bd5bd7..0dd94b9f0d 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -61,7 +61,7 @@ public: } }; -class BigSchedulerLocking { +class InterruptsBigSchedulerLocking { public: bool Lock(int* lockable) { @@ -80,11 +80,11 @@ public: }; class InterruptsBigSchedulerLocker : - public AutoLocker { + public AutoLocker { public: InterruptsBigSchedulerLocker() : - AutoLocker(&fState, false, true) + AutoLocker(&fState, false, true) { } @@ -157,10 +157,10 @@ enqueue(Thread* thread, bool newOne) targetCPU = &gCPUEntries[thread->previous_cpu->cpu_num]; } else if (gSingleCore) targetCore = &gCoreEntries[0]; - else if (threadData->GetCore() != NULL + else if (threadData->Core() != NULL && (!newOne || !threadData->HasCacheExpired()) && !threadData->ShouldRebalance()) { - targetCore = threadData->GetCore(); + targetCore = threadData->Core(); } bool rescheduleNeeded = threadData->ChooseCoreAndCPU(targetCore, targetCPU); @@ -194,7 +194,12 @@ enqueue(Thread* thread, bool newOne) void scheduler_enqueue_in_run_queue(Thread *thread) { - InterruptsSchedulerModeLocker _; +#if KDEBUG + if (are_interrupts_enabled()) + panic("scheduler_enqueue_in_run_queue: called with interrupts enabled"); +#endif + + SchedulerModeLocker _; TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); @@ -213,6 +218,11 @@ scheduler_enqueue_in_run_queue(Thread *thread) int32 scheduler_set_thread_priority(Thread *thread, int32 priority) { +#if KDEBUG + if (!are_interrupts_enabled()) + panic("scheduler_set_thread_priority: called with interrupts disabled"); +#endif + InterruptsSpinLocker _(thread->scheduler_lock); SchedulerModeLocker modeLocker; @@ -230,12 +240,12 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) if (thread->state != B_THREAD_READY) { if (thread->state == B_THREAD_RUNNING) { - ASSERT(threadData->GetCore() != NULL); + ASSERT(threadData->Core() != NULL); ASSERT(thread->cpu != NULL); CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num]; - SpinLocker coreLocker(threadData->GetCore()->fCPULock); + SpinLocker coreLocker(threadData->Core()->fCPULock); cpu->UpdatePriority(priority); } @@ -497,7 +507,7 @@ reschedule(int32 nextState) NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, oldThread, nextThread); - ASSERT(nextThreadData->GetCore() == core); + ASSERT(nextThreadData->Core() == core); nextThread->state = B_THREAD_RUNNING; // update CPU heap @@ -540,6 +550,11 @@ reschedule(int32 nextState) void scheduler_reschedule(int32 nextState) { +#if KDEBUG + if (are_interrupts_enabled()) + panic("scheduler_reschedule: called with interrupts enabled"); +#endif + if (!sSchedulerEnabled) { Thread* thread = thread_get_current_thread(); if (thread != NULL && nextState != B_THREAD_READY) @@ -588,7 +603,7 @@ scheduler_on_thread_destroy(Thread* thread) thread. Interrupts must be disabled and will be disabled when returning. */ void -scheduler_start(void) +scheduler_start() { InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); @@ -621,7 +636,7 @@ unassign_thread(Thread* thread, void* data) { CoreEntry* core = static_cast(data); - if (thread->scheduler_data->GetCore() == core + if (thread->scheduler_data->Core() == core && thread->pinned_to_cpu == 0) { thread->scheduler_data->UnassignCore(); } @@ -631,6 +646,11 @@ unassign_thread(Thread* thread, void* data) void scheduler_set_cpu_enabled(int32 cpuID, bool enabled) { +#if KDEBUG + if (are_interrupts_enabled()) + panic("scheduler_set_cpu_enabled: called with interrupts enabled"); +#endif + dprintf("scheduler: %s CPU %" B_PRId32 "\n", enabled ? "enabling" : "disabling", cpuID); @@ -667,12 +687,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) gCoreLoadHeap.RemoveMinimum(); } - package->fIdleCores.Remove(core); - package->fIdleCoreCount--; - package->fCoreCount--; - - if (package->fCoreCount == 0) - gIdlePackageList.Remove(package); + package->RemoveIdleCore(core); // get rid of threads thread_map(unassign_thread, core); @@ -689,7 +704,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) threadData->fWentSleepCount = -1; } - ASSERT(threadData->GetCore() == NULL); + ASSERT(threadData->Core() == NULL); enqueue(threadData->GetThread(), false); } } else if (oldCPUCount == 0) { @@ -701,12 +716,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) core->fHighLoad = false; gCoreLoadHeap.Insert(core, 0); - package->fCoreCount++; - package->fIdleCoreCount++; - package->fIdleCores.Add(core); - - if (package->fCoreCount == 1) - gIdlePackageList.Add(package); + package->AddIdleCore(core); } if (enabled) { @@ -846,11 +856,8 @@ init() new(&gIdlePackageList) IdlePackageList; - for (int32 i = 0; i < packageCount; i++) { - gPackageEntries[i].fPackageID = i; - gPackageEntries[i].fCoreCount = coreCount / packageCount; - gIdlePackageList.Insert(&gPackageEntries[i]); - } + for (int32 i = 0; i < packageCount; i++) + gPackageEntries[i].Init(i); for (int32 i = 0; i < coreCount; i++) { gCoreEntries[i].fCoreID = i; @@ -869,10 +876,8 @@ init() gCPUEntries[i].fCore = core; core->fPackage = package; - if (core->fCPUHeap.PeekMaximum() == NULL) { - package->fIdleCoreCount++; - package->fIdleCores.Insert(core); - } + if (core->fCPUHeap.PeekMaximum() == NULL) + package->AddIdleCore(core); result = core->fCPUHeap.Insert(&gCPUEntries[i], B_IDLE_PRIORITY); if (result != B_OK) @@ -888,7 +893,7 @@ init() void -scheduler_init(void) +scheduler_init() { int32 cpuCount = smp_get_num_cpus(); dprintf("scheduler_init: found %" B_PRId32 " logical cpu%s and %" B_PRId32 @@ -914,7 +919,7 @@ scheduler_init(void) void -scheduler_enable_scheduling(void) +scheduler_enable_scheduling() { sSchedulerEnabled = true; } @@ -972,7 +977,7 @@ _user_estimate_max_scheduling_latency(thread_id id) BReference threadReference(thread, true); ThreadData* threadData = thread->scheduler_data; - CoreEntry* core = threadData->GetCore(); + CoreEntry* core = threadData->Core(); if (core == NULL) core = &gCoreEntries[get_random() % gCoreCount]; @@ -1003,7 +1008,7 @@ _user_set_scheduler_mode(int32 mode) int32 -_user_get_scheduler_mode(void) +_user_get_scheduler_mode() { return gCurrentModeID; } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 75490dd30b..02c96917f2 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -45,7 +45,7 @@ const int kLoadDifference = kMaxLoad * 20 / 100; extern bool gSingleCore; -void init_debug_commands(void); +void init_debug_commands(); } // namespace Scheduler diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 8c9b0489c4..f4d55ebe4d 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -16,6 +16,13 @@ using namespace Scheduler; +class Scheduler::DebugDumper { +public: + static void DumpIdleCoresInPackage(PackageEntry* package); + +}; + + static CPUPriorityHeap sDebugCPUHeap; static CoreLoadHeap sDebugCoreHeap; @@ -67,37 +74,10 @@ CPUEntry::UpdatePriority(int32 priority) return; PackageEntry* packageEntry = fCore->fPackage; - if (maxPriority == B_IDLE_PRIORITY) { - WriteSpinLocker _(packageEntry->fCoreLock); - - // core goes idle - ASSERT(packageEntry->fIdleCoreCount >= 0); - ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount++; - packageEntry->fIdleCores.Add(fCore); - - if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { - // package goes idle - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList.Add(packageEntry); - } - } else if (corePriority == B_IDLE_PRIORITY) { - WriteSpinLocker _(packageEntry->fCoreLock); - - // core wakes up - ASSERT(packageEntry->fIdleCoreCount > 0); - ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount--; - packageEntry->fIdleCores.Remove(fCore); - - if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { - // package wakes up - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList.Remove(packageEntry); - } - } + if (maxPriority == B_IDLE_PRIORITY) + packageEntry->CoreGoesIdle(fCore); + else if (corePriority == B_IDLE_PRIORITY) + packageEntry->CoreWakesUp(fCore); } @@ -371,6 +351,94 @@ PackageEntry::PackageEntry() } +void +PackageEntry::Init(int32 id) +{ + fPackageID = id; +} + + +inline void +PackageEntry::CoreGoesIdle(CoreEntry* core) +{ + WriteSpinLocker _(fCoreLock); + + ASSERT(fIdleCoreCount >= 0); + ASSERT(fIdleCoreCount < fCoreCount); + + fIdleCoreCount++; + fIdleCores.Add(core); + + if (fIdleCoreCount == fCoreCount) { + // package goes idle + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Add(this); + } +} + + +inline void +PackageEntry::CoreWakesUp(CoreEntry* core) +{ + WriteSpinLocker _(fCoreLock); + + ASSERT(fIdleCoreCount > 0); + ASSERT(fIdleCoreCount <= fCoreCount); + + fIdleCoreCount--; + fIdleCores.Remove(core); + + if (fIdleCoreCount + 1 == fCoreCount) { + // package wakes up + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Remove(this); + } +} + + +void +PackageEntry::AddIdleCore(CoreEntry* core) +{ + fCoreCount++; + fIdleCoreCount++; + fIdleCores.Add(core); + + if (fCoreCount == 1) + gIdlePackageList.Add(this); +} + + +void +PackageEntry::RemoveIdleCore(CoreEntry* core) +{ + fIdleCores.Remove(core); + fIdleCoreCount--; + fCoreCount--; + + if (fCoreCount == 0) + gIdlePackageList.Remove(this); +} + + +/* static */ void +DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) +{ + kprintf("%-7" B_PRId32 " ", package->fPackageID); + + DoublyLinkedList::ReverseIterator iterator + = package->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); +} + + static int dump_run_queue(int argc, char **argv) { @@ -429,22 +497,8 @@ dump_idle_cores(int argc, char** argv) if (idleIterator.HasNext()) { kprintf("package cores\n"); - while (idleIterator.HasNext()) { - PackageEntry* entry = idleIterator.Next(); - kprintf("%-7" B_PRId32 " ", entry->fPackageID); - - DoublyLinkedList::ReverseIterator iterator - = entry->fIdleCores.GetReverseIterator(); - if (iterator.HasNext()) { - while (iterator.HasNext()) { - CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, - iterator.HasNext() ? ", " : ""); - } - } else - kprintf("-"); - kprintf("\n"); - } + while (idleIterator.HasNext()) + DebugDumper::DumpIdleCoresInPackage(idleIterator.Next()); } else kprintf("No idle packages.\n"); @@ -452,7 +506,7 @@ dump_idle_cores(int argc, char** argv) } -void Scheduler::init_debug_commands(void) +void Scheduler::init_debug_commands() { new(&sDebugCPUHeap) CPUPriorityHeap(smp_get_num_cpus()); new(&sDebugCoreHeap) CoreLoadHeap(smp_get_num_cpus()); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 4d02a646dc..be3721352e 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -21,6 +21,8 @@ namespace Scheduler { +class DebugDumper; + struct ThreadData; struct CPUEntry; @@ -122,15 +124,32 @@ public: // packages can go to the deep state of sleep). The heap stores only packages // with at least one core active and one core idle. The packages with all cores // idle are stored in gPackageIdleList (in LIFO manner). -struct PackageEntry : public DoublyLinkedListLinkImpl { +class PackageEntry : public DoublyLinkedListLinkImpl { +public: PackageEntry(); + void Init(int32 id); + + inline void CoreGoesIdle(CoreEntry* core); + inline void CoreWakesUp(CoreEntry* core); + + inline CoreEntry* GetIdleCore() const; + + void AddIdleCore(CoreEntry* core); + void RemoveIdleCore(CoreEntry* core); + + static inline PackageEntry* GetMostIdlePackage(); + static inline PackageEntry* GetLeastIdlePackage(); + +private: int32 fPackageID; DoublyLinkedList fIdleCores; int32 fIdleCoreCount; int32 fCoreCount; rw_spinlock fCoreLock; + + friend class DebugDumper; } CACHE_LINE_ALIGN; typedef DoublyLinkedList IdlePackageList; @@ -163,6 +182,48 @@ CoreEntry::GetCore(int32 cpu) } +inline CoreEntry* +PackageEntry::GetIdleCore() const +{ + return fIdleCores.Last(); +} + + +/* static */ inline PackageEntry* +PackageEntry::GetMostIdlePackage() +{ + PackageEntry* current = &gPackageEntries[0]; + for (int32 i = 1; i < gPackageCount; i++) { + if (gPackageEntries[i].fIdleCoreCount > current->fIdleCoreCount) + current = &gPackageEntries[i]; + } + + if (current->fIdleCoreCount == 0) + return NULL; + + return current; +} + + +/* static */ inline PackageEntry* +PackageEntry::GetLeastIdlePackage() +{ + PackageEntry* package = NULL; + + for (int32 i = 0; i < gPackageCount; i++) { + PackageEntry* current = &gPackageEntries[i]; + + int32 currentIdleCoreCount = current->fIdleCoreCount; + if (currentIdleCoreCount != 0 && (package == NULL + || currentIdleCoreCount < package->fIdleCoreCount)) { + package = current; + } + } + + return package; +} + + } // namespace Scheduler diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 378befc2fa..09670dd160 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -21,7 +21,7 @@ struct scheduler_mode_operations { bigtime_t maximum_latency; - void (*switch_to_mode)(void); + void (*switch_to_mode)(); void (*set_cpu_enabled)(int32 cpu, bool enabled); bool (*has_cache_expired)( const Scheduler::ThreadData* threadData); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 5f4ab2f12f..7ebf59133e 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -53,7 +53,7 @@ public: inline Thread* GetThread() const { return fThread; } inline int32 GetLoad() const { return fLoad; } - inline CoreEntry* GetCore() const { return fCore; } + inline CoreEntry* Core() const { return fCore; } inline void UnassignCore() { fCore = NULL; } bigtime_t fStolenTime; From 5a69ffc01e97cca974f9e113dcce130f9ae84aa7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 03:51:02 +0100 Subject: [PATCH 218/273] scheduler: Remove RunQueue::PeekSecondMaximum() --- src/system/kernel/scheduler/RunQueue.h | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index 4ecda7227a..9d54a3ad79 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -83,8 +83,7 @@ public: inline status_t GetInitStatus(); - inline Element* PeekMaximum(); - inline Element* PeekSecondMaximum(); + inline Element* PeekMaximum() const; inline void PushFront(Element* element, unsigned int priority); inline void PushBack(Element* elementt, unsigned int priority); @@ -251,7 +250,7 @@ RUN_QUEUE_CLASS_NAME::GetInitStatus() RUN_QUEUE_TEMPLATE_LIST Element* -RUN_QUEUE_CLASS_NAME::PeekMaximum() +RUN_QUEUE_CLASS_NAME::PeekMaximum() const { int priority = fBitmap.GetHighestSet(); if (priority < 0) @@ -271,22 +270,6 @@ RUN_QUEUE_CLASS_NAME::PeekMaximum() } -RUN_QUEUE_TEMPLATE_LIST -Element* -RUN_QUEUE_CLASS_NAME::PeekSecondMaximum() -{ - int priority = fBitmap.GetHighestSet(); - if (priority < 0) - return NULL; - - fBitmap.Clear(priority); - Element* element = PeekMaximum(); - fBitmap.Set(priority); - - return element; -} - - RUN_QUEUE_TEMPLATE_LIST void RUN_QUEUE_CLASS_NAME::PushFront(Element* element, From 3309bf33c7a43fe718fe1e5707f50750753422c1 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 03:52:13 +0100 Subject: [PATCH 219/273] kernel/util: Make MinMaxHeap::Peek*() const --- headers/private/kernel/util/MinMaxHeap.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index cc4ab0a7a0..40fd1b1c4c 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -73,8 +73,8 @@ public: MinMaxHeap(int initialSize); ~MinMaxHeap(); - inline Element* PeekMinimum(); - inline Element* PeekMaximum(); + inline Element* PeekMinimum() const; + inline Element* PeekMaximum() const; static const Key& GetKey(Element* element); @@ -190,7 +190,7 @@ MIN_MAX_HEAP_CLASS_NAME::~MinMaxHeap() MIN_MAX_HEAP_TEMPLATE_LIST Element* -MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() +MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() const { if (fMinLastElement > 0) return fMinElements[0]; @@ -205,7 +205,7 @@ MIN_MAX_HEAP_CLASS_NAME::PeekMinimum() MIN_MAX_HEAP_TEMPLATE_LIST Element* -MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() +MIN_MAX_HEAP_CLASS_NAME::PeekMaximum() const { if (fMaxLastElement > 0) return fMaxElements[0]; From e1e7235c60d942d4fd58ac7caedf4a9715efcc7a Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 18:15:31 +0100 Subject: [PATCH 220/273] scheduler: Encapsulate CoreEntry fields --- src/system/kernel/scheduler/low_latency.cpp | 12 +- src/system/kernel/scheduler/power_saving.cpp | 8 +- src/system/kernel/scheduler/scheduler.cpp | 139 ++++---------- src/system/kernel/scheduler/scheduler_cpu.cpp | 173 +++++++++++++++--- src/system/kernel/scheduler/scheduler_cpu.h | 151 ++++++++++++++- .../kernel/scheduler/scheduler_thread.cpp | 15 +- .../kernel/scheduler/scheduler_thread.h | 50 +++-- 7 files changed, 363 insertions(+), 185 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 441a609a7d..74cddd6801 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -37,13 +37,7 @@ has_cache_expired(const ThreadData* threadData) CoreEntry* core = threadData->Core(); - bigtime_t activeTime; - uint32 count; - do { - count = acquire_read_seqlock(&core->fActiveTimeLock); - activeTime = core->fActiveTime; - } while (!release_read_seqlock(&core->fActiveTimeLock, count)); - + bigtime_t activeTime = core->GetActiveTime(); return activeTime - threadData->fWentSleepActive > kCacheExpire; } @@ -138,9 +132,7 @@ rebalance_irqs(bool idle) other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); - SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; - cpuLocker.Unlock(); + int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; ASSERT(other != NULL); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 2ca77db7ec..d6f3a7d5bf 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -164,9 +164,7 @@ pack_irqs() irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - ReadSpinLocker coreLocker(gCoreHeapsLock); - int32 newCPU = smallTaskCore->fCPUHeap.PeekMinimum()->fCPUNumber; - coreLocker.Unlock(); + int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber; if (newCPU != cpu->cpu_num) assign_io_interrupt_to_cpu(irq->irq, newCPU); @@ -209,9 +207,7 @@ rebalance_irqs(bool idle) coreLocker.Unlock(); if (other == NULL) return; - SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; - cpuLocker.Unlock(); + int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); if (other == core) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 0dd94b9f0d..97e2e5256d 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -92,6 +92,11 @@ private: int fState; }; +class ThreadEnqueuer : public ThreadProcessing { +public: + void operator()(ThreadData* thread); +}; + scheduler_mode gCurrentModeID; scheduler_mode_operations* gCurrentMode; @@ -133,6 +138,16 @@ static int32* sCPUToCore; static int32* sCPUToPackage; +static void enqueue(Thread* thread, bool newOne); + + +void +ThreadEnqueuer::operator()(ThreadData* thread) +{ + enqueue(thread->GetThread(), false); +} + + void scheduler_dump_thread_data(Thread* thread) { @@ -245,7 +260,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) ASSERT(thread->cpu != NULL); CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num]; - SpinLocker coreLocker(threadData->Core()->fCPULock); + CoreCPUHeapLocker _(threadData->Core()); cpu->UpdatePriority(priority); } @@ -467,7 +482,7 @@ reschedule(int32 nextState) ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { if (!thread_is_idle_thread(oldThread)) { - SpinLocker runQueueLocker(core->fQueueLock); + CoreRunQueueLocker _(core); nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY); cpu->fRunQueue.Remove(nextThreadData); @@ -483,7 +498,7 @@ reschedule(int32 nextState) } Thread* nextThread = nextThreadData->GetThread(); - SpinLocker cpuLocker(core->fCPULock); + CoreCPUHeapLocker cpuLocker(core); cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); cpuLocker.Unlock(); @@ -511,9 +526,9 @@ reschedule(int32 nextState) nextThread->state = B_THREAD_RUNNING; // update CPU heap - SpinLocker coreLocker(core->fCPULock); + cpuLocker.Lock(); cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); - coreLocker.Unlock(); + cpuLocker.Unlock(); // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); @@ -631,18 +646,6 @@ scheduler_set_operation_mode(scheduler_mode mode) } -static void -unassign_thread(Thread* thread, void* data) -{ - CoreEntry* core = static_cast(data); - - if (thread->scheduler_data->Core() == core - && thread->pinned_to_cpu == 0) { - thread->scheduler_data->UnassignCore(); - } -} - - void scheduler_set_cpu_enabled(int32 cpuID, bool enabled) { @@ -660,78 +663,21 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) CPUEntry* cpu = &gCPUEntries[cpuID]; CoreEntry* core = cpu->fCore; - PackageEntry* package = core->fPackage; - int32 oldCPUCount = core->fCPUCount; + int32 oldCPUCount = core->CPUCount(); ASSERT(oldCPUCount >= 0); - if (enabled) - core->fCPUCount++; - else { + if (enabled) { + cpu->fLoad = 0; + core->AddCPU(cpu); + } else { cpu->UpdatePriority(B_IDLE_PRIORITY); - core->fCPUCount--; + + ThreadEnqueuer enqueuer; + core->RemoveCPU(cpu, enqueuer); } gCPU[cpuID].disabled = !enabled; - if (core->fCPUCount == 0) { - // core has been disabled - ASSERT(!enabled); - - if (core->fHighLoad) { - gCoreHighLoadHeap.ModifyKey(core, -1); - ASSERT(gCoreHighLoadHeap.PeekMinimum() == core); - gCoreHighLoadHeap.RemoveMinimum(); - } else { - gCoreLoadHeap.ModifyKey(core, -1); - ASSERT(gCoreLoadHeap.PeekMinimum() == core); - gCoreLoadHeap.RemoveMinimum(); - } - - package->RemoveIdleCore(core); - - // get rid of threads - thread_map(unassign_thread, core); - - core->fThreadCount = 0; - while (core->fRunQueue.PeekMaximum() != NULL) { - ThreadData* threadData = core->fRunQueue.PeekMaximum(); - - core->fRunQueue.Remove(threadData); - threadData->fEnqueued = false; - - if (threadData->fWentSleepCount == 0) { - core->fThreadList.Remove(threadData); - threadData->fWentSleepCount = -1; - } - - ASSERT(threadData->Core() == NULL); - enqueue(threadData->GetThread(), false); - } - } else if (oldCPUCount == 0) { - // core has been reenabled - ASSERT(enabled); - - cpu->fLoad = 0; - core->fLoad = 0; - core->fHighLoad = false; - gCoreLoadHeap.Insert(core, 0); - - package->AddIdleCore(core); - } - - if (enabled) { - core->fCPUHeap.Insert(cpu, B_IDLE_PRIORITY); - cpu->fLoad = 0; - } else { - core->fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); - ASSERT(core->fCPUHeap.PeekMaximum() == cpu); - core->fCPUHeap.RemoveMaximum(); - - ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); - core->fLoad -= cpu->fLoad; - ASSERT(core->fLoad >= 0); - } - if (!enabled) { cpu_ent* entry = &gCPU[cpuID]; @@ -856,32 +802,17 @@ init() new(&gIdlePackageList) IdlePackageList; - for (int32 i = 0; i < packageCount; i++) - gPackageEntries[i].Init(i); - - for (int32 i = 0; i < coreCount; i++) { - gCoreEntries[i].fCoreID = i; - gCoreEntries[i].fCPUCount = cpuCount / coreCount; - - result = gCoreLoadHeap.Insert(&gCoreEntries[i], 0); - if (result != B_OK) - return result; - } - for (int32 i = 0; i < cpuCount; i++) { CoreEntry* core = &gCoreEntries[sCPUToCore[i]]; PackageEntry* package = &gPackageEntries[sCPUToPackage[i]]; + package->Init(sCPUToPackage[i]); + core->Init(sCPUToCore[i], package); + gCPUEntries[i].fCPUNumber = i; gCPUEntries[i].fCore = core; - core->fPackage = package; - if (core->fCPUHeap.PeekMaximum() == NULL) - package->AddIdleCore(core); - - result = core->fCPUHeap.Insert(&gCPUEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; + core->AddCPU(&gCPUEntries[i]); } packageEntriesDeleter.Detach(); @@ -981,9 +912,9 @@ _user_estimate_max_scheduling_latency(thread_id id) if (core == NULL) core = &gCoreEntries[get_random() % gCoreCount]; - int32 threadCount = core->fThreadCount; - if (core->fCPUCount > 0) - threadCount /= core->fCPUCount; + int32 threadCount = core->ThreadCount(); + if (core->CPUCount() > 0) + threadCount /= core->CPUCount(); if (threadData->GetEffectivePriority() > 0) { threadCount -= threadCount * THREAD_MAX_SET_PRIORITY diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index f4d55ebe4d..896447e765 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -18,6 +18,7 @@ using namespace Scheduler; class Scheduler::DebugDumper { public: + static void DumpCoreRunQueue(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); }; @@ -63,17 +64,18 @@ CPUEntry::UpdatePriority(int32 priority) if (gCPU[fCPUNumber].disabled) return; - int32 corePriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); - fCore->fCPUHeap.ModifyKey(this, priority); + CPUPriorityHeap* cpuHeap = fCore->CPUHeap(); + int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); + cpuHeap->ModifyKey(this, priority); if (gSingleCore) return; - int32 maxPriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); + int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); if (corePriority == maxPriority) return; - PackageEntry* packageEntry = fCore->fPackage; + PackageEntry* packageEntry = fCore->Package(); if (maxPriority == B_IDLE_PRIORITY) packageEntry->CoreGoesIdle(fCore); else if (corePriority == B_IDLE_PRIORITY) @@ -93,9 +95,7 @@ CPUEntry::ComputeLoad() if (oldLoad != fLoad) { int32 delta = fLoad - oldLoad; - atomic_add(&fCore->fLoad, delta); - - fCore->UpdateLoad(); + fCore->UpdateLoad(delta); } if (fLoad > kVeryHighLoad) @@ -106,9 +106,9 @@ CPUEntry::ComputeLoad() ThreadData* CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) { - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); - ThreadData* sharedThread = fCore->fRunQueue.PeekMaximum(); + ThreadData* sharedThread = fCore->PeekThread(); ThreadData* pinnedThread = fRunQueue.PeekMaximum(); ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); @@ -132,16 +132,7 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) if (sharedPriority > pinnedPriority) { sharedThread->fEnqueued = false; - fCore->fRunQueue.Remove(sharedThread); - if (thread_is_idle_thread(sharedThread->GetThread()) - || fCore->fThreadList.Head() == sharedThread) { - atomic_add(&fCore->fStarvationCounter, 1); - } - - if (sharedThread->fWentSleepCount == 0) - fCore->fThreadList.Remove(sharedThread); - - atomic_add(&fCore->fThreadCount, -1); + fCore->Remove(sharedThread, sharedThread->fWentSleepCount == 0); return sharedThread; } @@ -259,15 +250,62 @@ CoreEntry::CoreEntry() void -CoreEntry::UpdateLoad() +CoreEntry::Init(int32 id, PackageEntry* package) { - ASSERT(!gSingleCore); + fCoreID = id; + fPackage = package; +} + +void +CoreEntry::PushFront(ThreadData* thread, int32 priority) +{ + fRunQueue.PushFront(thread, priority); + atomic_add(&fThreadCount, 1); +} + + +void +CoreEntry::PushBack(ThreadData* thread, int32 priority) +{ + fRunQueue.PushBack(thread, priority); + fThreadList.Insert(thread); + + atomic_add(&fThreadCount, 1); +} + + +void +CoreEntry::Remove(ThreadData* thread, bool starving) +{ + if (thread_is_idle_thread(thread->GetThread()) + || fThreadList.Head() == thread) { + atomic_add(&fStarvationCounter, 1); + } + if (starving) + fThreadList.Remove(thread); + fRunQueue.Remove(thread); + atomic_add(&fThreadCount, -1); +} + + +inline ThreadData* +CoreEntry::PeekThread() const +{ + return fRunQueue.PeekMaximum(); +} + + +void +CoreEntry::UpdateLoad(int32 delta) +{ if (fCPUCount == 0) { fLoad = 0; return; } + atomic_add(&fLoad, delta); + WriteSpinLocker coreLocker(gCoreHeapsLock); int32 newKey = GetLoad(); @@ -310,6 +348,81 @@ CoreEntry::UpdateLoad() } +void +CoreEntry::AddCPU(CPUEntry* cpu) +{ + ASSERT(fCPUCount >= 0); + if (fCPUCount++ == 0) { + // core has been reenabled + fLoad = 0; + fHighLoad = false; + gCoreLoadHeap.Insert(this, 0); + + fPackage->AddIdleCore(this); + } + + fCPUHeap.Insert(cpu, B_IDLE_PRIORITY); +} + + +void +CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) +{ + ASSERT(fCPUCount > 0); + if (--fCPUCount == 0) { + // core has been disabled + if (fHighLoad) { + gCoreHighLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); + gCoreHighLoadHeap.RemoveMinimum(); + } else { + gCoreLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == this); + gCoreLoadHeap.RemoveMinimum(); + } + + fPackage->RemoveIdleCore(this); + + // get rid of threads + thread_map(CoreEntry::_UnassignThread, this); + + fThreadCount = 0; + while (fRunQueue.PeekMaximum() != NULL) { + ThreadData* threadData = fRunQueue.PeekMaximum(); + + fRunQueue.Remove(threadData); + threadData->fEnqueued = false; + + if (threadData->fWentSleepCount == 0) + fThreadList.Remove(threadData); + threadData->fWentSleepCount = -1; + + ASSERT(threadData->Core() == NULL); + threadPostProcessing(threadData); + } + } + + fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); + ASSERT(fCPUHeap.PeekMaximum() == cpu); + fCPUHeap.RemoveMaximum(); + + ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); + fLoad -= cpu->fLoad; + ASSERT(fLoad >= 0); +} + + +/* static */ void +CoreEntry::_UnassignThread(Thread* thread, void* data) +{ + CoreEntry* core = static_cast(data); + ThreadData* threadData = thread->scheduler_data; + + if (threadData->Core() == core && thread->pinned_to_cpu == 0) + threadData->UnassignCore(); +} + + CoreLoadHeap::CoreLoadHeap(int32 coreCount) : MinMaxHeap(coreCount) @@ -323,7 +436,7 @@ CoreLoadHeap::Dump() CoreEntry* entry = PeekMinimum(); while (entry) { int32 key = GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, + kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(), entry->GetLoad() / 10); RemoveMinimum(); @@ -420,6 +533,13 @@ PackageEntry::RemoveIdleCore(CoreEntry* core) } +/* static */ void +DebugDumper::DumpCoreRunQueue(CoreEntry* core) +{ + core->fRunQueue.Dump(); +} + + /* static */ void DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) { @@ -430,7 +550,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) if (iterator.HasNext()) { while (iterator.HasNext()) { CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + kprintf("%" B_PRId32 "%s", coreEntry->ID(), iterator.HasNext() ? ", " : ""); } } else @@ -445,10 +565,9 @@ dump_run_queue(int argc, char **argv) int32 cpuCount = smp_get_num_cpus(); int32 coreCount = gCoreCount; - for (int32 i = 0; i < coreCount; i++) { kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); - gCoreEntries[i].fRunQueue.Dump(); + DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]); } for (int32 i = 0; i < cpuCount; i++) { @@ -476,11 +595,11 @@ dump_cpu_heap(int argc, char** argv) gCoreHighLoadHeap.Dump(); for (int32 i = 0; i < gCoreCount; i++) { - if (gCoreEntries[i].fCPUCount < 2) + if (gCoreEntries[i].CPUCount() < 2) continue; kprintf("\nCore %" B_PRId32 " heap:\n", i); - gCoreEntries[i].fCPUHeap.Dump(); + gCoreEntries[i].CPUHeap()->Dump(); } return 0; diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index be3721352e..6df157ea13 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -24,6 +25,7 @@ namespace Scheduler { class DebugDumper; struct ThreadData; +class ThreadProcessing; struct CPUEntry; struct CoreEntry; @@ -77,15 +79,57 @@ public: void Dump(); }; -struct CoreEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { +class CoreEntry : public MinMaxHeapLinkImpl, + public DoublyLinkedListLinkImpl { +public: CoreEntry(); + void Init(int32 id, PackageEntry* package); + + inline int32 ID() const { return fCoreID; } + inline PackageEntry* Package() const { return fPackage; } + inline int32 CPUCount() const + { return fCPUCount; } + + inline void LockCPUHeap(); + inline void UnlockCPUHeap(); + + inline CPUPriorityHeap* CPUHeap(); + + inline int32 ThreadCount() const + { return fThreadCount; } + + inline void LockRunQueue(); + inline void UnlockRunQueue(); + + void PushFront(ThreadData* thread, + int32 priority); + void PushBack(ThreadData* thread, + int32 priority); + void Remove(ThreadData* thread, + bool starving); + inline ThreadData* PeekThread() const; + + inline bigtime_t GetActiveTime() const; + inline void IncreaseActiveTime( + bigtime_t activeTime); + inline int32 GetLoad() const; - void UpdateLoad(); + void UpdateLoad(int32 delta); + + inline int32 StarvationCounter() const; + + void AddCPU(CPUEntry* cpu); + void RemoveCPU(CPUEntry* cpu, + ThreadProcessing& + threadPostProcessing); static inline CoreEntry* GetCore(int32 cpu); +private: + static void _UnassignThread(Thread* thread, + void* core); + int32 fCoreID; PackageEntry* fPackage; @@ -101,12 +145,46 @@ struct CoreEntry : public MinMaxHeapLinkImpl, spinlock fQueueLock; bigtime_t fActiveTime; - seqlock fActiveTimeLock; + mutable seqlock fActiveTimeLock; int32 fLoad; bool fHighLoad; + + friend class DebugDumper; } CACHE_LINE_ALIGN; +class CoreRunQueueLocking { +public: + inline bool Lock(CoreEntry* core) + { + core->LockRunQueue(); + return true; + } + + inline void Unlock(CoreEntry* core) + { + core->UnlockRunQueue(); + } +}; + +typedef AutoLocker CoreRunQueueLocker; + +class CoreCPUHeapLocking { +public: + inline bool Lock(CoreEntry* core) + { + core->LockCPUHeap(); + return true; + } + + inline void Unlock(CoreEntry* core) + { + core->UnlockCPUHeap(); + } +}; + +typedef AutoLocker CoreCPUHeapLocker; + class CoreLoadHeap : public MinMaxHeap { public: CoreLoadHeap() { } @@ -167,6 +245,64 @@ extern rw_spinlock gIdlePackageLock; extern int32 gPackageCount; +inline void +CoreEntry::LockCPUHeap() +{ + acquire_spinlock(&fCPULock); +} + + +inline void +CoreEntry::UnlockCPUHeap() +{ + release_spinlock(&fCPULock); +} + + +inline CPUPriorityHeap* +CoreEntry::CPUHeap() +{ + return &fCPUHeap; +} + + +inline void +CoreEntry::LockRunQueue() +{ + acquire_spinlock(&fQueueLock); +} + + +inline void +CoreEntry::UnlockRunQueue() +{ + release_spinlock(&fQueueLock); +} + + +inline void +CoreEntry::IncreaseActiveTime(bigtime_t activeTime) +{ + WriteSequentialLocker _(fActiveTimeLock); + fActiveTime += activeTime; +} + + +inline bigtime_t +CoreEntry::GetActiveTime() const +{ + bigtime_t activeTime; + + uint32 count; + do { + count = acquire_read_seqlock(&fActiveTimeLock); + activeTime = fActiveTime; + } while (!release_read_seqlock(&fActiveTimeLock, count)); + + return activeTime; +} + + inline int32 CoreEntry::GetLoad() const { @@ -175,6 +311,13 @@ CoreEntry::GetLoad() const } +inline int32 +CoreEntry::StarvationCounter() const +{ + return fStarvationCounter; +} + + /* static */ inline CoreEntry* CoreEntry::GetCore(int32 cpu) { diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index e1f280e1c7..5b933a5fbe 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -65,7 +65,7 @@ ThreadData::Dump() const kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount); kprintf("\tcore:\t\t\t%" B_PRId32 "\n", - fCore != NULL ? fCore->fCoreID : -1); + fCore != NULL ? fCore->ID() : -1); if (fCore != NULL && HasCacheExpired()) kprintf("\tcache affinity has expired\n"); } @@ -108,7 +108,7 @@ ThreadData::ComputeQuantum() quantum += fStolenTime; fStolenTime = 0; - int32 threadCount = (fCore->fThreadCount + 1) / fCore->fCPUCount; + int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount(); threadCount = max_c(threadCount, 1); quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum); @@ -137,7 +137,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const if (fThread->previous_cpu != NULL) { CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; if (previousCPU->fCore == core) { - SpinLocker cpuLocker(core->fCPULock); + CoreCPUHeapLocker _(core); if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { previousCPU->UpdatePriority(threadPriority); rescheduleNeeded = true; @@ -146,8 +146,8 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const } } - SpinLocker cpuLocker(core->fCPULock); - CPUEntry* cpu = core->fCPUHeap.PeekMinimum(); + CoreCPUHeapLocker _(core); + CPUEntry* cpu = core->CPUHeap()->PeekMinimum(); ASSERT(cpu != NULL); if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { @@ -195,3 +195,8 @@ ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, return maxQuantum - result; } + +ThreadProcessing::~ThreadProcessing() +{ +} + diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 7ebf59133e..2a49b68925 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -93,6 +93,13 @@ private: CoreEntry* fCore; }; +class ThreadProcessing { +public: + virtual ~ThreadProcessing(); + + virtual void operator()(ThreadData* thread) = 0; +}; + inline bool ThreadData::HasCacheExpired() const @@ -166,7 +173,7 @@ ThreadData::ShouldCancelPenalty() const if (fCore == NULL) return false; - return atomic_get(&fCore->fStarvationCounter) != fWentSleepCount + return fCore->StarvationCounter() != fWentSleepCount && system_time() - fWentSleep > gCurrentMode->base_quantum; } @@ -177,13 +184,8 @@ ThreadData::GoesAway() fLastInterruptTime = 0; fWentSleep = system_time(); - fWentSleepCount = atomic_get(&fCore->fStarvationCounter); - - uint32 count; - do { - count = acquire_read_seqlock(&fCore->fActiveTimeLock); - fWentSleepActive = fCore->fActiveTime; - } while (!release_read_seqlock(&fCore->fActiveTimeLock, count)); + fWentSleepCount = fCore->StarvationCounter(); + fWentSleepActive = fCore->GetActiveTime(); } @@ -195,7 +197,7 @@ ThreadData::PutBack() int32 priority = GetEffectivePriority(); - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); ASSERT(!fEnqueued); fEnqueued = true; if (fThread->pinned_to_cpu > 0) { @@ -203,10 +205,9 @@ ThreadData::PutBack() CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num]; cpu->fRunQueue.PushFront(this, priority); - } else { - fCore->fRunQueue.PushFront(this, priority); - atomic_add(&fCore->fThreadCount, 1); - } + } else + fCore->PushFront(this, priority); + fCore->UnlockRunQueue(); } @@ -219,7 +220,7 @@ ThreadData::Enqueue() int32 priority = GetEffectivePriority(); - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); ASSERT(!fEnqueued); fEnqueued = true; if (fThread->pinned_to_cpu > 0) { @@ -227,19 +228,15 @@ ThreadData::Enqueue() CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; cpu->fRunQueue.PushBack(this, priority); - } else { - fCore->fRunQueue.PushBack(this, priority); - fCore->fThreadList.Insert(this); - - atomic_add(&fCore->fThreadCount, 1); - } + } else + fCore->PushBack(this, priority); } inline bool ThreadData::Dequeue() { - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; @@ -250,12 +247,8 @@ ThreadData::Dequeue() CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; cpu->fRunQueue.Remove(this); } else { - fCore->fRunQueue.Remove(this); - ASSERT(fWentSleepCount < 1); - if (fWentSleepCount == 0) - fCore->fThreadList.Remove(this); - atomic_add(&fCore->fThreadCount, -1); + fCore->Remove(this, fWentSleepCount == 0); } return true; @@ -268,9 +261,8 @@ ThreadData::UpdateActivity(bigtime_t active) fMeasureActiveTime += active; gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - WriteSequentialLocker locker(fCore->fActiveTimeLock); - fCore->fActiveTime += active; - locker.Unlock(); + fCore->IncreaseActiveTime(active); + } From a08b40d4087b35c586959dc7da44035171d4cf15 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 20:12:03 +0100 Subject: [PATCH 221/273] scheduler: Encapsulate CPUEntry fields --- src/system/kernel/scheduler/low_latency.cpp | 2 +- src/system/kernel/scheduler/power_saving.cpp | 4 +- src/system/kernel/scheduler/scheduler.cpp | 91 +++++++++------ src/system/kernel/scheduler/scheduler_cpu.cpp | 107 +++++++++++++++--- src/system/kernel/scheduler/scheduler_cpu.h | 84 +++++++++++++- .../kernel/scheduler/scheduler_thread.cpp | 4 +- .../kernel/scheduler/scheduler_thread.h | 16 ++- 7 files changed, 236 insertions(+), 72 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 74cddd6801..dd6399c526 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -132,7 +132,7 @@ rebalance_irqs(bool idle) other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); - int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; + int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); ASSERT(other != NULL); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index d6f3a7d5bf..33bd769ad3 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -164,7 +164,7 @@ pack_irqs() irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber; + int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID(); if (newCPU != cpu->cpu_num) assign_io_interrupt_to_cpu(irq->irq, newCPU); @@ -207,7 +207,7 @@ rebalance_irqs(bool idle) coreLocker.Unlock(); if (other == NULL) return; - int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; + int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); if (other == core) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 97e2e5256d..a4797efc09 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -39,26 +39,60 @@ namespace Scheduler { -class SchedulerModeLocker : public ReadSpinLocker { +class SchedulerModeLocking { +public: + bool Lock(int* /* lockable */) + { + CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); + return true; + } + + void Unlock(int* /* lockable */) + { + CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); + } +}; + +class SchedulerModeLocker : + public AutoLocker { public: SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) : - ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) + AutoLocker(NULL, alreadyLocked, + lockIfNotLocked) { } }; -class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { +class InterruptsSchedulerModeLocking { +public: + bool Lock(int* lockable) + { + *lockable = disable_interrupts(); + CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); + return true; + } + + void Unlock(int* lockable) + { + CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); + restore_interrupts(*lockable); + } +}; + +class InterruptsSchedulerModeLocker : + public AutoLocker { public: InterruptsSchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) : - InterruptsReadSpinLocker( - gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) + AutoLocker(&fState, alreadyLocked, + lockIfNotLocked) { } + +private: + int fState; }; class InterruptsBigSchedulerLocking { @@ -67,14 +101,14 @@ public: { *lockable = disable_interrupts(); for (int32 i = 0; i < smp_get_num_cpus(); i++) - acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + CPUEntry::GetCPU(i)->LockScheduler(); return true; } void Unlock(int* lockable) { for (int32 i = 0; i < smp_get_num_cpus(); i++) - release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + CPUEntry::GetCPU(i)->UnlockScheduler(); restore_interrupts(*lockable); } }; @@ -193,10 +227,10 @@ enqueue(Thread* thread, bool newOne) if (threadPriority > heapPriority || (threadPriority == heapPriority && rescheduleNeeded)) { - if (targetCPU->fCPUNumber == smp_get_current_cpu()) - gCPU[targetCPU->fCPUNumber].invoke_scheduler = true; + if (targetCPU->ID() == smp_get_current_cpu()) + gCPU[targetCPU->ID()].invoke_scheduler = true; else { - smp_send_ici(targetCPU->fCPUNumber, SMP_MSG_RESCHEDULE, 0, 0, 0, + smp_send_ici(targetCPU->ID(), SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); } } @@ -484,8 +518,8 @@ reschedule(int32 nextState) if (!thread_is_idle_thread(oldThread)) { CoreRunQueueLocker _(core); - nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY); - cpu->fRunQueue.Remove(nextThreadData); + nextThreadData = cpu->PeekIdleThread(); + cpu->Remove(nextThreadData); nextThreadData->fEnqueued = false; putOldThreadAtBack = oldThread->pinned_to_cpu == 0; @@ -662,14 +696,13 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) gCurrentMode->set_cpu_enabled(cpuID, enabled); CPUEntry* cpu = &gCPUEntries[cpuID]; - CoreEntry* core = cpu->fCore; + CoreEntry* core = cpu->Core(); int32 oldCPUCount = core->CPUCount(); ASSERT(oldCPUCount >= 0); - if (enabled) { - cpu->fLoad = 0; - core->AddCPU(cpu); - } else { + if (enabled) + cpu->Start(); + else { cpu->UpdatePriority(B_IDLE_PRIORITY); ThreadEnqueuer enqueuer; @@ -679,21 +712,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) gCPU[cpuID].disabled = !enabled; if (!enabled) { - cpu_ent* entry = &gCPU[cpuID]; - - // get rid of irqs - SpinLocker locker(entry->irqs_lock); - irq_assignment* irq - = (irq_assignment*)list_get_first_item(&entry->irqs); - while (irq != NULL) { - locker.Unlock(); - - assign_io_interrupt_to_cpu(irq->irq, -1); - - locker.Lock(); - irq = (irq_assignment*)list_get_first_item(&entry->irqs); - } - locker.Unlock(); + cpu->Stop(); // don't wait until the thread quantum ends if (smp_get_current_cpu() != cpuID) { @@ -808,9 +827,7 @@ init() package->Init(sCPUToPackage[i]); core->Init(sCPUToCore[i], package); - - gCPUEntries[i].fCPUNumber = i; - gCPUEntries[i].fCore = core; + gCPUEntries[i].Init(i, core); core->AddCPU(&gCPUEntries[i]); } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 896447e765..0899a9b4d0 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -18,6 +18,7 @@ using namespace Scheduler; class Scheduler::DebugDumper { public: + static void DumpCPURunQueue(CPUEntry* cpu); static void DumpCoreRunQueue(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); @@ -58,6 +59,78 @@ CPUEntry::CPUEntry() } +void +CPUEntry::Init(int32 id, CoreEntry* core) +{ + fCPUNumber = id; + fCore = core; +} + + +void +CPUEntry::Start() +{ + fLoad = 0; + fCore->AddCPU(this); +} + + +void +CPUEntry::Stop() +{ + cpu_ent* entry = &gCPU[fCPUNumber]; + + // get rid of irqs + SpinLocker locker(entry->irqs_lock); + irq_assignment* irq + = (irq_assignment*)list_get_first_item(&entry->irqs); + while (irq != NULL) { + locker.Unlock(); + + assign_io_interrupt_to_cpu(irq->irq, -1); + + locker.Lock(); + irq = (irq_assignment*)list_get_first_item(&entry->irqs); + } + locker.Unlock(); +} + + +void +CPUEntry::PushFront(ThreadData* thread, int32 priority) +{ + fRunQueue.PushFront(thread, priority); +} + + +void +CPUEntry::PushBack(ThreadData* thread, int32 priority) +{ + fRunQueue.PushBack(thread, priority); +} + + +void +CPUEntry::Remove(ThreadData* thread) +{ + fRunQueue.Remove(thread); +} + + +inline ThreadData* +CPUEntry::PeekThread() const +{ + return fRunQueue.PeekMaximum(); +} + + +ThreadData* +CPUEntry::PeekIdleThread() const +{ + return fRunQueue.GetHead(B_IDLE_PRIORITY); +} + + void CPUEntry::UpdatePriority(int32 priority) { @@ -213,10 +286,10 @@ CPUPriorityHeap::Dump() kprintf("cpu priority load\n"); CPUEntry* entry = PeekMinimum(); while (entry) { - int32 cpu = entry->fCPUNumber; + int32 cpu = entry->ID(); int32 key = GetKey(entry); kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, - entry->fLoad / 10); + entry->GetLoad() / 10); RemoveMinimum(); sDebugCPUHeap.Insert(entry, key); @@ -406,8 +479,8 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) ASSERT(fCPUHeap.PeekMaximum() == cpu); fCPUHeap.RemoveMaximum(); - ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); - fLoad -= cpu->fLoad; + ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad); + fLoad -= cpu->GetLoad(); ASSERT(fLoad >= 0); } @@ -533,6 +606,19 @@ PackageEntry::RemoveIdleCore(CoreEntry* core) } +/* static */ void +DebugDumper::DumpCPURunQueue(CPUEntry* cpu) +{ + ThreadRunQueue::ConstIterator iterator = cpu->fRunQueue.GetConstIterator(); + + if (iterator.HasNext() + && !thread_is_idle_thread(iterator.Next()->GetThread())) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", cpu->ID()); + cpu->fRunQueue.Dump(); + } +} + + /* static */ void DebugDumper::DumpCoreRunQueue(CoreEntry* core) { @@ -570,17 +656,8 @@ dump_run_queue(int argc, char **argv) DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]); } - for (int32 i = 0; i < cpuCount; i++) { - CPUEntry* cpu = &gCPUEntries[i]; - ThreadRunQueue::ConstIterator iterator - = cpu->fRunQueue.GetConstIterator(); - - if (iterator.HasNext() - && !thread_is_idle_thread(iterator.Next()->GetThread())) { - kprintf("\nCPU %" B_PRId32 " run queue:\n", i); - cpu->fRunQueue.Dump(); - } - } + for (int32 i = 0; i < cpuCount; i++) + DebugDumper::DumpCPURunQueue(&gCPUEntries[i]); return 0; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 6df157ea13..9480619ad0 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -40,11 +40,38 @@ public: void Dump() const; }; -struct CPUEntry : public MinMaxHeapLinkImpl { +class CPUEntry : public MinMaxHeapLinkImpl { +public: CPUEntry(); + void Init(int32 id, CoreEntry* core); + + inline int32 ID() const { return fCPUNumber; } + inline CoreEntry* Core() const { return fCore; } + + void Start(); + void Stop(); + + inline void EnterScheduler(); + inline void ExitScheduler(); + + inline void LockScheduler(); + inline void UnlockScheduler(); + + void PushFront(ThreadData* thread, + int32 priority); + void PushBack(ThreadData* thread, + int32 priority); + void Remove(ThreadData* thread); + inline ThreadData* PeekThread() const; + ThreadData* PeekIdleThread() const; + void UpdatePriority(int32 priority); + inline void IncreaseActiveTime( + bigtime_t activeTime); + + inline int32 GetLoad() const { return fLoad; } void ComputeLoad(); ThreadData* ChooseNextThread(ThreadData* oldThread, @@ -53,6 +80,12 @@ struct CPUEntry : public MinMaxHeapLinkImpl { void TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData); + static inline CPUEntry* GetCPU(int32 cpu); + +private: + inline void _RequestPerformanceLevel( + ThreadData* threadData); + int32 fCPUNumber; CoreEntry* fCore; @@ -65,10 +98,7 @@ struct CPUEntry : public MinMaxHeapLinkImpl { bigtime_t fMeasureActiveTime; bigtime_t fMeasureTime; -private: - inline void _RequestPerformanceLevel( - ThreadData* threadData); - + friend class DebugDumper; } CACHE_LINE_ALIGN; class CPUPriorityHeap : public MinMaxHeap { @@ -245,6 +275,48 @@ extern rw_spinlock gIdlePackageLock; extern int32 gPackageCount; +inline void +CPUEntry::EnterScheduler() +{ + acquire_read_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::ExitScheduler() +{ + release_read_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::LockScheduler() +{ + acquire_write_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::UnlockScheduler() +{ + release_write_spinlock(&fSchedulerModeLock); +} + + +inline void +CPUEntry::IncreaseActiveTime(bigtime_t activeTime) +{ + fMeasureActiveTime += activeTime; +} + + +/* static */ inline CPUEntry* +CPUEntry::GetCPU(int32 cpu) +{ + return &gCPUEntries[cpu]; +} + + inline void CoreEntry::LockCPUHeap() { @@ -321,7 +393,7 @@ CoreEntry::StarvationCounter() const /* static */ inline CoreEntry* CoreEntry::GetCore(int32 cpu) { - return gCPUEntries[cpu].fCore; + return gCPUEntries[cpu].Core(); } diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 5b933a5fbe..3bcb4d79be 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -77,7 +77,7 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) bool rescheduleNeeded = false; if (targetCore == NULL && targetCPU != NULL) - targetCore = targetCPU->fCore; + targetCore = targetCPU->Core(); else if (targetCore != NULL && targetCPU == NULL) targetCPU = _ChooseCPU(targetCore, rescheduleNeeded); else if (targetCore == NULL && targetCPU == NULL) { @@ -136,7 +136,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const if (fThread->previous_cpu != NULL) { CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; - if (previousCPU->fCore == core) { + if (previousCPU->Core() == core) { CoreCPUHeapLocker _(core); if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { previousCPU->UpdatePriority(threadPriority); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 2a49b68925..f0d06903c1 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -203,8 +203,8 @@ ThreadData::PutBack() if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->cpu != NULL); - CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num]; - cpu->fRunQueue.PushFront(this, priority); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num); + cpu->PushFront(this, priority); } else fCore->PushFront(this, priority); fCore->UnlockRunQueue(); @@ -226,8 +226,8 @@ ThreadData::Enqueue() if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); - CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; - cpu->fRunQueue.PushBack(this, priority); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + cpu->PushBack(this, priority); } else fCore->PushBack(this, priority); } @@ -244,8 +244,8 @@ ThreadData::Dequeue() if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); - CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; - cpu->fRunQueue.Remove(this); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + cpu->Remove(this); } else { ASSERT(fWentSleepCount < 1); fCore->Remove(this, fWentSleepCount == 0); @@ -259,10 +259,8 @@ inline void ThreadData::UpdateActivity(bigtime_t active) { fMeasureActiveTime += active; - gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - + CPUEntry::GetCPU(smp_get_current_cpu())->IncreaseActiveTime(active); fCore->IncreaseActiveTime(active); - } From b24ea642d759ad6e6b30007cb112b3cdfad35204 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 21:32:21 +0100 Subject: [PATCH 222/273] scheduler: Encapsulate ThreadData fields --- src/system/kernel/scheduler/low_latency.cpp | 2 +- src/system/kernel/scheduler/power_saving.cpp | 2 +- src/system/kernel/scheduler/scheduler.cpp | 8 ++- src/system/kernel/scheduler/scheduler_cpu.cpp | 27 +++++----- src/system/kernel/scheduler/scheduler_cpu.h | 3 +- .../kernel/scheduler/scheduler_thread.h | 52 ++++++++++++++----- 6 files changed, 58 insertions(+), 36 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index dd6399c526..d4090b7855 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -38,7 +38,7 @@ has_cache_expired(const ThreadData* threadData) CoreEntry* core = threadData->Core(); bigtime_t activeTime = core->GetActiveTime(); - return activeTime - threadData->fWentSleepActive > kCacheExpire; + return activeTime - threadData->WentSleepActive() > kCacheExpire; } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 33bd769ad3..8248c6b9f4 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -41,7 +41,7 @@ has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - return system_time() - threadData->fWentSleep > kCacheExpire; + return system_time() - threadData->WentSleep() > kCacheExpire; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index a4797efc09..87bfc789f8 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -476,8 +476,8 @@ reschedule(int32 nextState) ThreadData* oldThreadData = oldThread->scheduler_data; // return time spent in interrupts - oldThreadData->fStolenTime - += gCPU[thisCPU].interrupt_time - oldThreadData->fLastInterruptTime; + oldThreadData->IncreaseStolenTime( + gCPU[thisCPU].interrupt_time - oldThreadData->LastInterruptTime()); bool enqueueOldThread = false; bool putOldThreadAtBack = false; @@ -520,7 +520,6 @@ reschedule(int32 nextState) nextThreadData = cpu->PeekIdleThread(); cpu->Remove(nextThreadData); - nextThreadData->fEnqueued = false; putOldThreadAtBack = oldThread->pinned_to_cpu == 0; } else @@ -581,8 +580,7 @@ reschedule(int32 nextState) add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); } else { - nextThreadData->fQuantumStart = system_time(); - + nextThreadData->StartQuantum(); gCurrentMode->rebalance_irqs(true); } diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 0899a9b4d0..fe4800bcf8 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -113,6 +113,8 @@ CPUEntry::PushBack(ThreadData* thread, int32 priority) void CPUEntry::Remove(ThreadData* thread) { + ASSERT(thread->IsEnqueued()); + thread->SetDequeued(); fRunQueue.Remove(thread); } @@ -203,14 +205,11 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) return oldThread; if (sharedPriority > pinnedPriority) { - sharedThread->fEnqueued = false; - - fCore->Remove(sharedThread, sharedThread->fWentSleepCount == 0); + fCore->Remove(sharedThread); return sharedThread; } - pinnedThread->fEnqueued = false; - fRunQueue.Remove(pinnedThread); + Remove(pinnedThread); return pinnedThread; } @@ -243,7 +242,7 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) cpuEntry->last_kernel_time = nextThread->kernel_time; cpuEntry->last_user_time = nextThread->user_time; - nextThreadData->fLastInterruptTime = cpuEntry->interrupt_time; + nextThreadData->SetLastInterruptTime(cpuEntry->interrupt_time); _RequestPerformanceLevel(nextThreadData); } @@ -349,13 +348,15 @@ CoreEntry::PushBack(ThreadData* thread, int32 priority) void -CoreEntry::Remove(ThreadData* thread, bool starving) +CoreEntry::Remove(ThreadData* thread) { + ASSERT(thread->IsEnqueued()); + thread->SetDequeued(); if (thread_is_idle_thread(thread->GetThread()) || fThreadList.Head() == thread) { atomic_add(&fStarvationCounter, 1); } - if (starving) + if (thread->WentSleepCount() == 0) fThreadList.Remove(thread); fRunQueue.Remove(thread); atomic_add(&fThreadCount, -1); @@ -459,20 +460,16 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) // get rid of threads thread_map(CoreEntry::_UnassignThread, this); - fThreadCount = 0; while (fRunQueue.PeekMaximum() != NULL) { ThreadData* threadData = fRunQueue.PeekMaximum(); - fRunQueue.Remove(threadData); - threadData->fEnqueued = false; - - if (threadData->fWentSleepCount == 0) - fThreadList.Remove(threadData); - threadData->fWentSleepCount = -1; + Remove(threadData); ASSERT(threadData->Core() == NULL); threadPostProcessing(threadData); } + + fThreadCount = 0; } fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 9480619ad0..f3a9d35601 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -136,8 +136,7 @@ public: int32 priority); void PushBack(ThreadData* thread, int32 priority); - void Remove(ThreadData* thread, - bool starving); + void Remove(ThreadData* thread); inline ThreadData* PeekThread() const; inline bigtime_t GetActiveTime() const; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index f0d06903c1..7a8b6163cf 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -38,7 +38,17 @@ public: bool ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU); + inline bigtime_t LastInterruptTime() const + { return fLastInterruptTime; } + inline void SetLastInterruptTime(bigtime_t interruptTime) + { fLastInterruptTime = interruptTime; } + + inline void IncreaseStolenTime(bigtime_t stolenTime); + inline void GoesAway(); + inline bigtime_t WentSleep() const { return fWentSleep; } + inline bigtime_t WentSleepActive() const { return fWentSleepActive; } + inline bigtime_t WentSleepCount() const { return fWentSleepCount; } inline void PutBack(); inline void Enqueue(); @@ -49,6 +59,10 @@ public: inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); bigtime_t ComputeQuantum(); + inline void StartQuantum(); + + inline bool IsEnqueued() const { return fEnqueued; } + inline void SetDequeued() { fEnqueued = false; } inline Thread* GetThread() const { return fThread; } inline int32 GetLoad() const { return fLoad; } @@ -56,16 +70,6 @@ public: inline CoreEntry* Core() const { return fCore; } inline void UnassignCore() { fCore = NULL; } - bigtime_t fStolenTime; - bigtime_t fQuantumStart; - bigtime_t fLastInterruptTime; - - bigtime_t fWentSleep; - bigtime_t fWentSleepActive; - int32 fWentSleepCount; - - bool fEnqueued; - private: inline int32 _GetPenalty() const; inline int32 _GetMinimalPriority() const; @@ -79,6 +83,16 @@ private: bigtime_t minQuantum, int32 maxPriority, int32 minPriority, int32 priority); + bigtime_t fStolenTime; + bigtime_t fQuantumStart; + bigtime_t fLastInterruptTime; + + bigtime_t fWentSleep; + bigtime_t fWentSleepActive; + int32 fWentSleepCount; + + bool fEnqueued; + Thread* fThread; int32 fPriorityPenalty; @@ -178,6 +192,13 @@ ThreadData::ShouldCancelPenalty() const } +inline void +ThreadData::IncreaseStolenTime(bigtime_t stolenTime) +{ + fStolenTime += stolenTime; +} + + inline void ThreadData::GoesAway() { @@ -240,7 +261,6 @@ ThreadData::Dequeue() if (!fEnqueued) return false; - fEnqueued = false; if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); @@ -248,9 +268,10 @@ ThreadData::Dequeue() cpu->Remove(this); } else { ASSERT(fWentSleepCount < 1); - fCore->Remove(this, fWentSleepCount == 0); + fCore->Remove(this); } + ASSERT(!fEnqueued); return true; } @@ -299,6 +320,13 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) } +inline void +ThreadData::StartQuantum() +{ + fQuantumStart = system_time(); +} + + inline int32 ThreadData::_GetPenalty() const { From ede552ab25e23e2aa64b6953c4ef848699266881 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 22:06:33 +0100 Subject: [PATCH 223/273] scheduler: Keep thread effective priority cached --- .../kernel/scheduler/scheduler_thread.cpp | 18 ++++++++++++++ .../kernel/scheduler/scheduler_thread.h | 24 +++++++++---------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 3bcb4d79be..ab45e78de0 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -22,6 +22,7 @@ ThreadData::Init() { fPriorityPenalty = 0; fAdditionalPenalty = 0; + fEffectivePriority = -1; fTimeLeft = 0; fStolenTime = 0; @@ -121,6 +122,23 @@ ThreadData::ComputeQuantum() } +void +ThreadData::_ComputeEffectivePriority() const +{ + if (thread_is_idle_thread(fThread)) + fEffectivePriority = B_IDLE_PRIORITY; + else if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + fEffectivePriority = fThread->priority; + else { + fEffectivePriority = fThread->priority; + fEffectivePriority -= _GetPenalty(); + + ASSERT(fEffectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(fEffectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + } +} + + inline CoreEntry* ThreadData::_ChooseCore() const { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 7a8b6163cf..4260e82586 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -74,6 +74,8 @@ private: inline int32 _GetPenalty() const; inline int32 _GetMinimalPriority() const; + void _ComputeEffectivePriority() const; + inline CoreEntry* _ChooseCore() const; inline CPUEntry* _ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const; @@ -98,6 +100,8 @@ private: int32 fPriorityPenalty; int32 fAdditionalPenalty; + mutable int32 fEffectivePriority; + bigtime_t fTimeLeft; bigtime_t fMeasureActiveTime; @@ -133,18 +137,9 @@ ThreadData::ShouldRebalance() const inline int32 ThreadData::GetEffectivePriority() const { - if (thread_is_idle_thread(fThread)) - return B_IDLE_PRIORITY; - if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return fThread->priority; - - int32 effectivePriority = fThread->priority; - effectivePriority -= _GetPenalty(); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; + if (fEffectivePriority == -1) + _ComputeEffectivePriority(); + return fEffectivePriority; } @@ -158,6 +153,7 @@ ThreadData::IncreasePenalty() TRACE("increasing thread %ld penalty\n", fThread->id); + fEffectivePriority = -1; int32 oldPenalty = fPriorityPenalty++; ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); @@ -173,8 +169,10 @@ ThreadData::IncreasePenalty() inline void ThreadData::CancelPenalty() { - if (fPriorityPenalty != 0) + if (fPriorityPenalty != 0) { TRACE("cancelling thread %ld penalty\n", fThread->id); + fEffectivePriority = -1; + } fAdditionalPenalty = 0; fPriorityPenalty = 0; From cf4984f64588ef80b96d573c0931c3517585c162 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 23 Dec 2013 22:49:12 +0100 Subject: [PATCH 224/273] scheduler: Use precomputed time slice lengths --- src/system/kernel/scheduler/scheduler.cpp | 7 +-- .../kernel/scheduler/scheduler_thread.cpp | 48 ++++++++++++------- .../kernel/scheduler/scheduler_thread.h | 3 ++ 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 87bfc789f8..567e8a5636 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -579,10 +579,9 @@ reschedule(int32 nextState) bigtime_t quantum = nextThreadData->ComputeQuantum(); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); - } else { - nextThreadData->StartQuantum(); + } else gCurrentMode->rebalance_irqs(true); - } + nextThreadData->StartQuantum(); modeLocker.Unlock(); if (nextThread != oldThread) @@ -674,6 +673,8 @@ scheduler_set_operation_mode(scheduler_mode mode) gCurrentMode = sSchedulerModes[mode]; gCurrentMode->switch_to_mode(); + ThreadData::ComputeQuantumLengths(); + return B_OK; } diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index ab45e78de0..70f540a94b 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -9,6 +9,9 @@ using namespace Scheduler; +bigtime_t Scheduler::gQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; + + ThreadData::ThreadData(Thread* thread) : fThread(thread) @@ -116,12 +119,37 @@ ThreadData::ComputeQuantum() quantum = std::max(quantum, gCurrentMode->minimal_quantum); fTimeLeft = quantum; - fQuantumStart = system_time(); return quantum; } +/* static */ void +ThreadData::ComputeQuantumLengths() +{ + for (int32 priority = 0; priority <= THREAD_MAX_SET_PRIORITY; priority++) { + const bigtime_t kQuantum0 = gCurrentMode->base_quantum; + if (priority >= B_URGENT_DISPLAY_PRIORITY) { + gQuantumLengths[priority] = kQuantum0; + continue; + } + + const bigtime_t kQuantum1 + = kQuantum0 * gCurrentMode->quantum_multipliers[0]; + if (priority > B_NORMAL_PRIORITY) { + gQuantumLengths[priority] = _ScaleQuantum(kQuantum1, kQuantum0, + B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, priority); + continue; + } + + const bigtime_t kQuantum2 + = kQuantum0 * gCurrentMode->quantum_multipliers[1]; + gQuantumLengths[priority] = _ScaleQuantum(kQuantum2, kQuantum1, + B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); + } +} + + void ThreadData::_ComputeEffectivePriority() const { @@ -181,23 +209,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const inline bigtime_t ThreadData::_GetBaseQuantum() const { - int32 priority = GetEffectivePriority(); - - const bigtime_t kQuantum0 = gCurrentMode->base_quantum; - if (priority >= B_URGENT_DISPLAY_PRIORITY) - return kQuantum0; - - const bigtime_t kQuantum1 - = kQuantum0 * gCurrentMode->quantum_multipliers[0]; - if (priority > B_NORMAL_PRIORITY) { - return _ScaleQuantum(kQuantum1, kQuantum0, B_URGENT_DISPLAY_PRIORITY, - B_NORMAL_PRIORITY, priority); - } - - const bigtime_t kQuantum2 - = kQuantum0 * gCurrentMode->quantum_multipliers[1]; - return _ScaleQuantum(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, - B_IDLE_PRIORITY, priority); + return gQuantumLengths[GetEffectivePriority()]; } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 4260e82586..7500bcbc7d 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -70,6 +70,7 @@ public: inline CoreEntry* Core() const { return fCore; } inline void UnassignCore() { fCore = NULL; } + static void ComputeQuantumLengths(); private: inline int32 _GetPenalty() const; inline int32 _GetMinimalPriority() const; @@ -118,6 +119,8 @@ public: virtual void operator()(ThreadData* thread) = 0; }; +extern bigtime_t gQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; + inline bool ThreadData::HasCacheExpired() const From ebe5420f845cae655b92303a8b664320307248fa Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 26 Dec 2013 19:36:39 +0100 Subject: [PATCH 225/273] scheduler: No need for gQuantumLengths to be global --- src/system/kernel/scheduler/scheduler_thread.cpp | 10 +++++----- src/system/kernel/scheduler/scheduler_thread.h | 2 -- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 70f540a94b..fb6c8aa54e 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -9,7 +9,7 @@ using namespace Scheduler; -bigtime_t Scheduler::gQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; +static bigtime_t sQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; ThreadData::ThreadData(Thread* thread) @@ -130,21 +130,21 @@ ThreadData::ComputeQuantumLengths() for (int32 priority = 0; priority <= THREAD_MAX_SET_PRIORITY; priority++) { const bigtime_t kQuantum0 = gCurrentMode->base_quantum; if (priority >= B_URGENT_DISPLAY_PRIORITY) { - gQuantumLengths[priority] = kQuantum0; + sQuantumLengths[priority] = kQuantum0; continue; } const bigtime_t kQuantum1 = kQuantum0 * gCurrentMode->quantum_multipliers[0]; if (priority > B_NORMAL_PRIORITY) { - gQuantumLengths[priority] = _ScaleQuantum(kQuantum1, kQuantum0, + sQuantumLengths[priority] = _ScaleQuantum(kQuantum1, kQuantum0, B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, priority); continue; } const bigtime_t kQuantum2 = kQuantum0 * gCurrentMode->quantum_multipliers[1]; - gQuantumLengths[priority] = _ScaleQuantum(kQuantum2, kQuantum1, + sQuantumLengths[priority] = _ScaleQuantum(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); } } @@ -209,7 +209,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const inline bigtime_t ThreadData::_GetBaseQuantum() const { - return gQuantumLengths[GetEffectivePriority()]; + return sQuantumLengths[GetEffectivePriority()]; } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 7500bcbc7d..022cf34eb1 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -119,8 +119,6 @@ public: virtual void operator()(ThreadData* thread) = 0; }; -extern bigtime_t gQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; - inline bool ThreadData::HasCacheExpired() const From 96dcc73b39cc68a59c276a35690f8af1886214ef Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 26 Dec 2013 18:30:37 +0100 Subject: [PATCH 226/273] scheduler: Add scheduler profiler A bit hackish implementation of a profiler for the scheduler. SCHEDULER_ENTER_FUNCTION at the begining of each function aren't nice and usage of __PRETTY_FUNCTION__ isn't any better (both gcc and clang support it though), but it was quick to implement and doesn't lose information on inlined functions. It's just a tool, not an integral part of the kernal anyway. --- src/system/kernel/Jamfile | 1 + src/system/kernel/scheduler/low_latency.cpp | 10 +- src/system/kernel/scheduler/power_saving.cpp | 16 +- src/system/kernel/scheduler/scheduler.cpp | 37 ++- src/system/kernel/scheduler/scheduler_cpu.cpp | 28 ++ src/system/kernel/scheduler/scheduler_cpu.h | 26 +- .../kernel/scheduler/scheduler_profiler.cpp | 298 ++++++++++++++++++ .../kernel/scheduler/scheduler_profiler.h | 135 ++++++++ .../kernel/scheduler/scheduler_thread.cpp | 16 + .../kernel/scheduler/scheduler_thread.h | 32 ++ 10 files changed, 580 insertions(+), 19 deletions(-) create mode 100644 src/system/kernel/scheduler/scheduler_profiler.cpp create mode 100644 src/system/kernel/scheduler/scheduler_profiler.h diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index ab80c5d31a..1ed3b2f278 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -66,6 +66,7 @@ KernelMergeObject kernel_core.o : power_saving.cpp scheduler.cpp scheduler_cpu.cpp + scheduler_profiler.cpp scheduler_thread.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index d4090b7855..f76673be88 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -9,6 +9,7 @@ #include "scheduler_common.h" #include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_profiler.h" #include "scheduler_thread.h" @@ -33,10 +34,9 @@ set_cpu_enabled(int32 /* cpu */, bool /* enabled */) static bool has_cache_expired(const ThreadData* threadData) { - ASSERT(!gSingleCore); + SCHEDULER_ENTER_FUNCTION(); CoreEntry* core = threadData->Core(); - bigtime_t activeTime = core->GetActiveTime(); return activeTime - threadData->WentSleepActive() > kCacheExpire; } @@ -45,6 +45,8 @@ has_cache_expired(const ThreadData* threadData) static CoreEntry* choose_core(const ThreadData* /* threadData */) { + SCHEDULER_ENTER_FUNCTION(); + // wake new package PackageEntry* package = gIdlePackageList.Last(); if (package == NULL) { @@ -72,6 +74,8 @@ choose_core(const ThreadData* /* threadData */) static bool should_rebalance(const ThreadData* threadData) { + SCHEDULER_ENTER_FUNCTION(); + int32 coreLoad = threadData->Core()->GetLoad(); // If the thread produces more than 50% of the load, leave it here. In @@ -104,6 +108,8 @@ should_rebalance(const ThreadData* threadData) static void rebalance_irqs(bool idle) { + SCHEDULER_ENTER_FUNCTION(); + if (idle) return; diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 8248c6b9f4..c1da765748 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -10,6 +10,7 @@ #include "scheduler_common.h" #include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_profiler.h" #include "scheduler_thread.h" @@ -39,8 +40,7 @@ set_cpu_enabled(int32 cpu, bool enabled) static bool has_cache_expired(const ThreadData* threadData) { - ASSERT(!gSingleCore); - + SCHEDULER_ENTER_FUNCTION(); return system_time() - threadData->WentSleep() > kCacheExpire; } @@ -48,6 +48,8 @@ has_cache_expired(const ThreadData* threadData) static CoreEntry* choose_small_task_core() { + SCHEDULER_ENTER_FUNCTION(); + ReadSpinLocker locker(gCoreHeapsLock); CoreEntry* core = gCoreLoadHeap.PeekMaximum(); locker.Unlock(); @@ -66,6 +68,8 @@ choose_small_task_core() static CoreEntry* choose_idle_core() { + SCHEDULER_ENTER_FUNCTION(); + PackageEntry* package = PackageEntry::GetLeastIdlePackage(); if (package == NULL) @@ -81,6 +85,8 @@ choose_idle_core() static CoreEntry* choose_core(const ThreadData* threadData) { + SCHEDULER_ENTER_FUNCTION(); + CoreEntry* core = NULL; // try to pack all threads on one core @@ -111,6 +117,8 @@ choose_core(const ThreadData* threadData) static bool should_rebalance(const ThreadData* threadData) { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(!gSingleCore); CoreEntry* core = threadData->Core(); @@ -151,6 +159,8 @@ should_rebalance(const ThreadData* threadData) static inline void pack_irqs() { + SCHEDULER_ENTER_FUNCTION(); + CoreEntry* smallTaskCore = atomic_pointer_get(&sSmallTaskCore); if (smallTaskCore == NULL) return; @@ -177,6 +187,8 @@ pack_irqs() static void rebalance_irqs(bool idle) { + SCHEDULER_ENTER_FUNCTION(); + if (idle && sSmallTaskCore != NULL) { pack_irqs(); return; diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 567e8a5636..824dbe4ace 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -32,6 +32,7 @@ #include "scheduler_common.h" #include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_profiler.h" #include "scheduler_thread.h" #include "scheduler_tracing.h" @@ -192,7 +193,7 @@ scheduler_dump_thread_data(Thread* thread) static void enqueue(Thread* thread, bool newOne) { - ASSERT(thread != NULL); + SCHEDULER_ENTER_FUNCTION(); ThreadData* threadData = thread->scheduler_data; @@ -243,10 +244,8 @@ enqueue(Thread* thread, bool newOne) void scheduler_enqueue_in_run_queue(Thread *thread) { -#if KDEBUG - if (are_interrupts_enabled()) - panic("scheduler_enqueue_in_run_queue: called with interrupts enabled"); -#endif + ASSERT(!are_interrupts_enabled()); + SCHEDULER_ENTER_FUNCTION(); SchedulerModeLocker _; @@ -267,14 +266,13 @@ scheduler_enqueue_in_run_queue(Thread *thread) int32 scheduler_set_thread_priority(Thread *thread, int32 priority) { -#if KDEBUG - if (!are_interrupts_enabled()) - panic("scheduler_set_thread_priority: called with interrupts disabled"); -#endif + ASSERT(are_interrupts_enabled()); InterruptsSpinLocker _(thread->scheduler_lock); SchedulerModeLocker modeLocker; + SCHEDULER_ENTER_FUNCTION(); + ThreadData* threadData = thread->scheduler_data; int32 oldPriority = thread->priority; @@ -431,6 +429,8 @@ switch_thread(Thread* fromThread, Thread* toThread) static inline void update_thread_times(Thread* oldThread, Thread* nextThread) { + SCHEDULER_ENTER_FUNCTION(); + bigtime_t now = system_time(); if (oldThread == nextThread) { SpinLocker _(oldThread->time_lock); @@ -459,6 +459,7 @@ static void reschedule(int32 nextState) { ASSERT(!are_interrupts_enabled()); + SCHEDULER_ENTER_FUNCTION(); SchedulerModeLocker modeLocker; @@ -584,6 +585,9 @@ reschedule(int32 nextState) nextThreadData->StartQuantum(); modeLocker.Unlock(); + + SCHEDULER_EXIT_FUNCTION(); + if (nextThread != oldThread) switch_thread(oldThread, nextThread); } @@ -596,10 +600,8 @@ reschedule(int32 nextState) void scheduler_reschedule(int32 nextState) { -#if KDEBUG - if (are_interrupts_enabled()) - panic("scheduler_reschedule: called with interrupts enabled"); -#endif + ASSERT(!are_interrupts_enabled()); + SCHEDULER_ENTER_FUNCTION(); if (!sSchedulerEnabled) { Thread* thread = thread_get_current_thread(); @@ -652,6 +654,7 @@ void scheduler_start() { InterruptsSpinLocker _(thread_get_current_thread()->scheduler_lock); + SCHEDULER_ENTER_FUNCTION(); reschedule(B_THREAD_READY); } @@ -847,6 +850,10 @@ scheduler_init() " cache level%s\n", cpuCount, cpuCount != 1 ? "s" : "", gCPUCacheLevelCount, gCPUCacheLevelCount != 1 ? "s" : ""); +#ifdef SCHEDULER_PROFILING + Profiling::Profiler::Initialize(); +#endif + status_t result = init(); if (result != B_OK) panic("scheduler_init: failed to initialize scheduler\n"); @@ -923,6 +930,10 @@ _user_estimate_max_scheduling_latency(thread_id id) } BReference threadReference(thread, true); +#ifdef SCHEDULER_PROFILING + InterruptsLocker _; +#endif + ThreadData* threadData = thread->scheduler_data; CoreEntry* core = threadData->Core(); if (core == NULL) diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index fe4800bcf8..f2030759f9 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -99,6 +99,7 @@ CPUEntry::Stop() void CPUEntry::PushFront(ThreadData* thread, int32 priority) { + SCHEDULER_ENTER_FUNCTION(); fRunQueue.PushFront(thread, priority); } @@ -106,6 +107,7 @@ CPUEntry::PushFront(ThreadData* thread, int32 priority) void CPUEntry::PushBack(ThreadData* thread, int32 priority) { + SCHEDULER_ENTER_FUNCTION(); fRunQueue.PushBack(thread, priority); } @@ -113,6 +115,7 @@ CPUEntry::PushBack(ThreadData* thread, int32 priority) void CPUEntry::Remove(ThreadData* thread) { + SCHEDULER_ENTER_FUNCTION(); ASSERT(thread->IsEnqueued()); thread->SetDequeued(); fRunQueue.Remove(thread); @@ -122,6 +125,7 @@ CPUEntry::Remove(ThreadData* thread) inline ThreadData* CPUEntry::PeekThread() const { + SCHEDULER_ENTER_FUNCTION(); return fRunQueue.PeekMaximum(); } @@ -129,6 +133,7 @@ CPUEntry::PeekThread() const ThreadData* CPUEntry::PeekIdleThread() const { + SCHEDULER_ENTER_FUNCTION(); return fRunQueue.GetHead(B_IDLE_PRIORITY); } @@ -136,6 +141,8 @@ CPUEntry::PeekIdleThread() const void CPUEntry::UpdatePriority(int32 priority) { + SCHEDULER_ENTER_FUNCTION(); + if (gCPU[fCPUNumber].disabled) return; @@ -161,6 +168,8 @@ CPUEntry::UpdatePriority(int32 priority) void CPUEntry::ComputeLoad() { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(!gSingleCore); ASSERT(fCPUNumber == smp_get_current_cpu()); @@ -181,6 +190,8 @@ CPUEntry::ComputeLoad() ThreadData* CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) { + SCHEDULER_ENTER_FUNCTION(); + CoreRunQueueLocker _(fCore); ThreadData* sharedThread = fCore->PeekThread(); @@ -217,6 +228,8 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) void CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) { + SCHEDULER_ENTER_FUNCTION(); + cpu_ent* cpuEntry = &gCPU[fCPUNumber]; Thread* oldThread = oldThreadData->GetThread(); @@ -252,6 +265,8 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) inline void CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) { + SCHEDULER_ENTER_FUNCTION(); + int32 load = std::max(threadData->GetLoad(), fCore->GetLoad()); load = std::min(std::max(load, int32(0)), kMaxLoad); @@ -332,6 +347,8 @@ CoreEntry::Init(int32 id, PackageEntry* package) void CoreEntry::PushFront(ThreadData* thread, int32 priority) { + SCHEDULER_ENTER_FUNCTION(); + fRunQueue.PushFront(thread, priority); atomic_add(&fThreadCount, 1); } @@ -340,6 +357,8 @@ CoreEntry::PushFront(ThreadData* thread, int32 priority) void CoreEntry::PushBack(ThreadData* thread, int32 priority) { + SCHEDULER_ENTER_FUNCTION(); + fRunQueue.PushBack(thread, priority); fThreadList.Insert(thread); @@ -350,6 +369,8 @@ CoreEntry::PushBack(ThreadData* thread, int32 priority) void CoreEntry::Remove(ThreadData* thread) { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(thread->IsEnqueued()); thread->SetDequeued(); if (thread_is_idle_thread(thread->GetThread()) @@ -366,6 +387,7 @@ CoreEntry::Remove(ThreadData* thread) inline ThreadData* CoreEntry::PeekThread() const { + SCHEDULER_ENTER_FUNCTION(); return fRunQueue.PeekMaximum(); } @@ -373,6 +395,8 @@ CoreEntry::PeekThread() const void CoreEntry::UpdateLoad(int32 delta) { + SCHEDULER_ENTER_FUNCTION(); + if (fCPUCount == 0) { fLoad = 0; return; @@ -544,6 +568,8 @@ PackageEntry::Init(int32 id) inline void PackageEntry::CoreGoesIdle(CoreEntry* core) { + SCHEDULER_ENTER_FUNCTION(); + WriteSpinLocker _(fCoreLock); ASSERT(fIdleCoreCount >= 0); @@ -563,6 +589,8 @@ PackageEntry::CoreGoesIdle(CoreEntry* core) inline void PackageEntry::CoreWakesUp(CoreEntry* core) { + SCHEDULER_ENTER_FUNCTION(); + WriteSpinLocker _(fCoreLock); ASSERT(fIdleCoreCount > 0); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index f3a9d35601..859627972b 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -17,6 +17,7 @@ #include "RunQueue.h" #include "scheduler_common.h" #include "scheduler_modes.h" +#include "scheduler_profiler.h" namespace Scheduler { @@ -277,6 +278,7 @@ extern int32 gPackageCount; inline void CPUEntry::EnterScheduler() { + SCHEDULER_ENTER_FUNCTION(); acquire_read_spinlock(&fSchedulerModeLock); } @@ -284,6 +286,7 @@ CPUEntry::EnterScheduler() inline void CPUEntry::ExitScheduler() { + SCHEDULER_ENTER_FUNCTION(); release_read_spinlock(&fSchedulerModeLock); } @@ -291,6 +294,7 @@ CPUEntry::ExitScheduler() inline void CPUEntry::LockScheduler() { + SCHEDULER_ENTER_FUNCTION(); acquire_write_spinlock(&fSchedulerModeLock); } @@ -298,6 +302,7 @@ CPUEntry::LockScheduler() inline void CPUEntry::UnlockScheduler() { + SCHEDULER_ENTER_FUNCTION(); release_write_spinlock(&fSchedulerModeLock); } @@ -305,6 +310,7 @@ CPUEntry::UnlockScheduler() inline void CPUEntry::IncreaseActiveTime(bigtime_t activeTime) { + SCHEDULER_ENTER_FUNCTION(); fMeasureActiveTime += activeTime; } @@ -312,6 +318,7 @@ CPUEntry::IncreaseActiveTime(bigtime_t activeTime) /* static */ inline CPUEntry* CPUEntry::GetCPU(int32 cpu) { + SCHEDULER_ENTER_FUNCTION(); return &gCPUEntries[cpu]; } @@ -319,6 +326,7 @@ CPUEntry::GetCPU(int32 cpu) inline void CoreEntry::LockCPUHeap() { + SCHEDULER_ENTER_FUNCTION(); acquire_spinlock(&fCPULock); } @@ -326,6 +334,7 @@ CoreEntry::LockCPUHeap() inline void CoreEntry::UnlockCPUHeap() { + SCHEDULER_ENTER_FUNCTION(); release_spinlock(&fCPULock); } @@ -333,6 +342,7 @@ CoreEntry::UnlockCPUHeap() inline CPUPriorityHeap* CoreEntry::CPUHeap() { + SCHEDULER_ENTER_FUNCTION(); return &fCPUHeap; } @@ -340,6 +350,7 @@ CoreEntry::CPUHeap() inline void CoreEntry::LockRunQueue() { + SCHEDULER_ENTER_FUNCTION(); acquire_spinlock(&fQueueLock); } @@ -347,6 +358,7 @@ CoreEntry::LockRunQueue() inline void CoreEntry::UnlockRunQueue() { + SCHEDULER_ENTER_FUNCTION(); release_spinlock(&fQueueLock); } @@ -354,6 +366,7 @@ CoreEntry::UnlockRunQueue() inline void CoreEntry::IncreaseActiveTime(bigtime_t activeTime) { + SCHEDULER_ENTER_FUNCTION(); WriteSequentialLocker _(fActiveTimeLock); fActiveTime += activeTime; } @@ -362,14 +375,14 @@ CoreEntry::IncreaseActiveTime(bigtime_t activeTime) inline bigtime_t CoreEntry::GetActiveTime() const { - bigtime_t activeTime; + SCHEDULER_ENTER_FUNCTION(); + bigtime_t activeTime; uint32 count; do { count = acquire_read_seqlock(&fActiveTimeLock); activeTime = fActiveTime; } while (!release_read_seqlock(&fActiveTimeLock, count)); - return activeTime; } @@ -377,6 +390,8 @@ CoreEntry::GetActiveTime() const inline int32 CoreEntry::GetLoad() const { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(fCPUCount >= 0); return fLoad / fCPUCount; } @@ -385,6 +400,7 @@ CoreEntry::GetLoad() const inline int32 CoreEntry::StarvationCounter() const { + SCHEDULER_ENTER_FUNCTION(); return fStarvationCounter; } @@ -392,6 +408,7 @@ CoreEntry::StarvationCounter() const /* static */ inline CoreEntry* CoreEntry::GetCore(int32 cpu) { + SCHEDULER_ENTER_FUNCTION(); return gCPUEntries[cpu].Core(); } @@ -399,6 +416,7 @@ CoreEntry::GetCore(int32 cpu) inline CoreEntry* PackageEntry::GetIdleCore() const { + SCHEDULER_ENTER_FUNCTION(); return fIdleCores.Last(); } @@ -406,6 +424,8 @@ PackageEntry::GetIdleCore() const /* static */ inline PackageEntry* PackageEntry::GetMostIdlePackage() { + SCHEDULER_ENTER_FUNCTION(); + PackageEntry* current = &gPackageEntries[0]; for (int32 i = 1; i < gPackageCount; i++) { if (gPackageEntries[i].fIdleCoreCount > current->fIdleCoreCount) @@ -422,6 +442,8 @@ PackageEntry::GetMostIdlePackage() /* static */ inline PackageEntry* PackageEntry::GetLeastIdlePackage() { + SCHEDULER_ENTER_FUNCTION(); + PackageEntry* package = NULL; for (int32 i = 0; i < gPackageCount; i++) { diff --git a/src/system/kernel/scheduler/scheduler_profiler.cpp b/src/system/kernel/scheduler/scheduler_profiler.cpp new file mode 100644 index 0000000000..e0746c1838 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_profiler.cpp @@ -0,0 +1,298 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + +#include "scheduler_profiler.h" + +#include +#include + +#include + + +#ifdef SCHEDULER_PROFILING + + +using namespace Scheduler; +using namespace Scheduler::Profiling; + + +static Profiler* sProfiler; + +static int dump_profiler(int argc, char** argv); + + +Profiler::Profiler() + : + kMaxFunctionEntries(1024), + kMaxFunctionStackEntries(512), + fFunctionData(new(std::nothrow) FunctionData[kMaxFunctionEntries]), + fStatus(B_OK) +{ + B_INITIALIZE_SPINLOCK(&fFunctionLock); + + if (fFunctionData == NULL) { + fStatus = B_NO_MEMORY; + return; + } + memset(fFunctionData, 0, sizeof(FunctionData) * kMaxFunctionEntries); + + for (int32 i = 0; i < smp_get_num_cpus(); i++) { + fFunctionStacks[i] + = new(std::nothrow) FunctionEntry[kMaxFunctionStackEntries]; + if (fFunctionStacks[i] == NULL) { + fStatus = B_NO_MEMORY; + return; + } + memset(fFunctionStacks[i], 0, + sizeof(FunctionEntry) * kMaxFunctionStackEntries); + } + memset(fFunctionStackPointers, 0, sizeof(int32) * smp_get_num_cpus()); +} + + +void +Profiler::EnterFunction(int32 cpu, const char* functionName) +{ + FunctionData* function = _FindFunction(functionName); + if (function == NULL) + return; + atomic_add((int32*)&function->fCalled, 1); + + FunctionEntry* stackEntry + = &fFunctionStacks[cpu][fFunctionStackPointers[cpu]]; + fFunctionStackPointers[cpu]++; + + ASSERT(fFunctionStackPointers[cpu] < kMaxFunctionStackEntries); + + stackEntry->fFunction = function; + stackEntry->fEntryTime = system_time(); + stackEntry->fOthersTime = 0; +} + + +void +Profiler::ExitFunction(int32 cpu, const char* functionName) +{ + ASSERT(fFunctionStackPointers[cpu] > 0); + fFunctionStackPointers[cpu]--; + FunctionEntry* stackEntry + = &fFunctionStacks[cpu][fFunctionStackPointers[cpu]]; + + bigtime_t timeSpent = system_time() - stackEntry->fEntryTime; + atomic_add64(&stackEntry->fFunction->fTimeInclusive, timeSpent); + atomic_add64(&stackEntry->fFunction->fTimeExclusive, + timeSpent - stackEntry->fOthersTime); + + if (fFunctionStackPointers[cpu] > 0) { + stackEntry = &fFunctionStacks[cpu][fFunctionStackPointers[cpu] - 1]; + stackEntry->fOthersTime += timeSpent; + } +} + + +void +Profiler::DumpCalled(uint32 maxCount) +{ + uint32 count = _FunctionCount(); + + qsort(fFunctionData, count, sizeof(FunctionData), + &_CompareFunctions); + + if (maxCount > 0) + count = std::min(count, maxCount); + _Dump(count); +} + + +void +Profiler::DumpTimeInclusive(uint32 maxCount) +{ + uint32 count = _FunctionCount(); + + qsort(fFunctionData, count, sizeof(FunctionData), + &_CompareFunctions); + + if (maxCount > 0) + count = std::min(count, maxCount); + _Dump(count); +} + + +void +Profiler::DumpTimeExclusive(uint32 maxCount) +{ + uint32 count = _FunctionCount(); + + qsort(fFunctionData, count, sizeof(FunctionData), + &_CompareFunctions); + + if (maxCount > 0) + count = std::min(count, maxCount); + _Dump(count); +} + + +void +Profiler::DumpTimeInclusivePerCall(uint32 maxCount) +{ + uint32 count = _FunctionCount(); + + qsort(fFunctionData, count, sizeof(FunctionData), + &_CompareFunctionsPerCall); + + if (maxCount > 0) + count = std::min(count, maxCount); + _Dump(count); +} + + +void +Profiler::DumpTimeExclusivePerCall(uint32 maxCount) +{ + uint32 count = _FunctionCount(); + + qsort(fFunctionData, count, sizeof(FunctionData), + &_CompareFunctionsPerCall); + + if (maxCount > 0) + count = std::min(count, maxCount); + _Dump(count); +} + + +/* static */ Profiler* +Profiler::Get() +{ + return sProfiler; +} + + +/* static */ void +Profiler::Initialize() +{ + sProfiler = new(std::nothrow) Profiler; + if (sProfiler == NULL || sProfiler->GetStatus() != B_OK) + panic("Scheduler::Profiling::Profiler: could not initialize profiler"); + + add_debugger_command_etc("scheduler_profiler", &dump_profiler, + "Show data collected by scheduler profiler", + "[ [ ] ]\n" + "Shows data collected by scheduler profiler\n" + " - Field used to sort functions. Available: called," + " time-inclusive, time-inclusive-per-call, time-exclusive," + " time-exclusive-per-call.\n" + " (defaults to \"called\")\n" + " - Maximum number of showed functions.\n", 0); +} + + +uint32 +Profiler::_FunctionCount() const +{ + uint32 count; + for (count = 0; count < kMaxFunctionEntries; count++) { + if (fFunctionData[count].fFunction == NULL) + break; + } + return count; +} + + +void +Profiler::_Dump(uint32 count) +{ + kprintf("Function calls (%" B_PRId32 " functions):\n", count); + kprintf(" called time-inclusive per-call time-exclusive per-call " + "function\n"); + for (uint32 i = 0; i < count; i++) { + FunctionData* function = &fFunctionData[i]; + kprintf("%10" B_PRId32 " %14" B_PRId64 " %8" B_PRId64 " %14" B_PRId64 + " %8" B_PRId64 " %s\n", function->fCalled, + function->fTimeInclusive, + function->fTimeInclusive / function->fCalled, + function->fTimeExclusive, + function->fTimeExclusive / function->fCalled, function->fFunction); + } +} + + +Profiler::FunctionData* +Profiler::_FindFunction(const char* function) +{ + for (uint32 i = 0; i < kMaxFunctionEntries; i++) { + if (fFunctionData[i].fFunction == NULL) + break; + if (!strcmp(fFunctionData[i].fFunction, function)) + return fFunctionData + i; + } + + SpinLocker _(fFunctionLock); + for (uint32 i = 0; i < kMaxFunctionEntries; i++) { + if (fFunctionData[i].fFunction == NULL) { + fFunctionData[i].fFunction = function; + return fFunctionData + i; + } + if (!strcmp(fFunctionData[i].fFunction, function)) + return fFunctionData + i; + } + + return NULL; +} + + +template +/* static */ int +Profiler::_CompareFunctions(const void* _a, const void* _b) +{ + const FunctionData* a = static_cast(_a); + const FunctionData* b = static_cast(_b); + + return b->*Member - a->*Member; +} + + +template +/* static */ int +Profiler::_CompareFunctionsPerCall(const void* _a, const void* _b) +{ + const FunctionData* a = static_cast(_a); + const FunctionData* b = static_cast(_b); + + return b->*Member / b->fCalled - a->*Member / a->fCalled; +} + + +static int +dump_profiler(int argc, char** argv) +{ + if (argc < 2) { + Profiler::Get()->DumpCalled(0); + return 0; + } + + int32 count = 0; + if (argc >= 3) + count = parse_expression(argv[2]); + count = std::max(count, int32(0)); + + if (!strcmp(argv[1], "called")) + Profiler::Get()->DumpCalled(count); + else if (!strcmp(argv[1], "time-inclusive")) + Profiler::Get()->DumpTimeInclusive(count); + else if (!strcmp(argv[1], "time-inclusive-per-call")) + Profiler::Get()->DumpTimeInclusivePerCall(count); + else if (!strcmp(argv[1], "time-exclusive")) + Profiler::Get()->DumpTimeExclusive(count); + else if (!strcmp(argv[1], "time-exclusive-per-call")) + Profiler::Get()->DumpTimeExclusivePerCall(count); + else + print_debugger_command_usage(argv[0]); + + return 0; +} + + +#endif // SCHEDULER_PROFILING + diff --git a/src/system/kernel/scheduler/scheduler_profiler.h b/src/system/kernel/scheduler/scheduler_profiler.h new file mode 100644 index 0000000000..fb87c160a8 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_profiler.h @@ -0,0 +1,135 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_PROFILER_H +#define KERNEL_SCHEDULER_PROFILER_H + + +#include + + +//#define SCHEDULER_PROFILING +#ifdef SCHEDULER_PROFILING + + +#define SCHEDULER_ENTER_FUNCTION() \ + Scheduler::Profiling::Function schedulerProfiler(__PRETTY_FUNCTION__) + +#define SCHEDULER_EXIT_FUNCTION() \ + schedulerProfiler.Exit() + + +namespace Scheduler { + +namespace Profiling { + +class Profiler { +public: + Profiler(); + + void EnterFunction(int32 cpu, const char* function); + void ExitFunction(int32 cpu, const char* function); + + void DumpCalled(uint32 count); + void DumpTimeInclusive(uint32 count); + void DumpTimeExclusive(uint32 count); + void DumpTimeInclusivePerCall(uint32 count); + void DumpTimeExclusivePerCall(uint32 count); + + status_t GetStatus() const { return fStatus; } + + static Profiler* Get(); + static void Initialize(); + +private: + struct FunctionData { + const char* fFunction; + + uint32 fCalled; + + bigtime_t fTimeInclusive; + bigtime_t fTimeExclusive; + }; + + struct FunctionEntry { + FunctionData* fFunction; + + bigtime_t fEntryTime; + bigtime_t fOthersTime; + }; + + uint32 _FunctionCount() const; + void _Dump(uint32 count); + + FunctionData* _FindFunction(const char* function); + + template + static int _CompareFunctions(const void* a, const void* b); + + template + static int _CompareFunctionsPerCall(const void* a, + const void* b); + + const uint32 kMaxFunctionEntries; + const uint32 kMaxFunctionStackEntries; + + FunctionEntry* fFunctionStacks[SMP_MAX_CPUS]; + uint32 fFunctionStackPointers[SMP_MAX_CPUS]; + + FunctionData* fFunctionData; + spinlock fFunctionLock; + + status_t fStatus; +}; + +class Function { +public: + inline Function(const char* functionName); + inline ~Function(); + + inline void Exit(); + +private: + const char* fFunctionName; +}; + + +Function::Function(const char* functionName) + : + fFunctionName(functionName) +{ + Profiler::Get()->EnterFunction(smp_get_current_cpu(), fFunctionName); +} + + +Function::~Function() +{ + if (fFunctionName != NULL) + Exit(); +} + + +void +Function::Exit() +{ + Profiler::Get()->ExitFunction(smp_get_current_cpu(), fFunctionName); + fFunctionName = NULL; +} + + +} // namespace Profiling + +} // namespace Scheduler + + +#else // SCHEDULER_PROFILING + +#define SCHEDULER_ENTER_FUNCTION() (void)0 +#define SCHEDULER_EXIT_FUNCTION() (void)0 + +#endif // !SCHEDULER_PROFILING + + +#endif // KERNEL_SCHEDULER_PROFILER_H + diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index fb6c8aa54e..de6a64bb75 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -78,6 +78,8 @@ ThreadData::Dump() const bool ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) { + SCHEDULER_ENTER_FUNCTION(); + bool rescheduleNeeded = false; if (targetCore == NULL && targetCPU != NULL) @@ -100,6 +102,8 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) bigtime_t ThreadData::ComputeQuantum() { + SCHEDULER_ENTER_FUNCTION(); + bigtime_t quantum; if (fTimeLeft != 0) quantum = fTimeLeft; @@ -127,6 +131,8 @@ ThreadData::ComputeQuantum() /* static */ void ThreadData::ComputeQuantumLengths() { + SCHEDULER_ENTER_FUNCTION(); + for (int32 priority = 0; priority <= THREAD_MAX_SET_PRIORITY; priority++) { const bigtime_t kQuantum0 = gCurrentMode->base_quantum; if (priority >= B_URGENT_DISPLAY_PRIORITY) { @@ -153,6 +159,8 @@ ThreadData::ComputeQuantumLengths() void ThreadData::_ComputeEffectivePriority() const { + SCHEDULER_ENTER_FUNCTION(); + if (thread_is_idle_thread(fThread)) fEffectivePriority = B_IDLE_PRIORITY; else if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) @@ -170,6 +178,8 @@ ThreadData::_ComputeEffectivePriority() const inline CoreEntry* ThreadData::_ChooseCore() const { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(!gSingleCore); return gCurrentMode->choose_core(this); } @@ -178,6 +188,8 @@ ThreadData::_ChooseCore() const inline CPUEntry* ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const { + SCHEDULER_ENTER_FUNCTION(); + int32 threadPriority = GetEffectivePriority(); if (fThread->previous_cpu != NULL) { @@ -209,6 +221,8 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const inline bigtime_t ThreadData::_GetBaseQuantum() const { + SCHEDULER_ENTER_FUNCTION(); + return sQuantumLengths[GetEffectivePriority()]; } @@ -217,6 +231,8 @@ ThreadData::_GetBaseQuantum() const ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, int32 maxPriority, int32 minPriority, int32 priority) { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(priority <= maxPriority); ASSERT(priority >= minPriority); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 022cf34eb1..3fb2db65e2 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -11,6 +11,7 @@ #include "scheduler_common.h" #include "scheduler_cpu.h" +#include "scheduler_profiler.h" namespace Scheduler { @@ -123,6 +124,7 @@ public: inline bool ThreadData::HasCacheExpired() const { + SCHEDULER_ENTER_FUNCTION(); return gCurrentMode->has_cache_expired(this); } @@ -130,6 +132,8 @@ ThreadData::HasCacheExpired() const inline bool ThreadData::ShouldRebalance() const { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(!gSingleCore); return gCurrentMode->should_rebalance(this); } @@ -138,6 +142,8 @@ ThreadData::ShouldRebalance() const inline int32 ThreadData::GetEffectivePriority() const { + SCHEDULER_ENTER_FUNCTION(); + if (fEffectivePriority == -1) _ComputeEffectivePriority(); return fEffectivePriority; @@ -147,6 +153,8 @@ ThreadData::GetEffectivePriority() const inline void ThreadData::IncreasePenalty() { + SCHEDULER_ENTER_FUNCTION(); + if (fThread->priority < B_LOWEST_ACTIVE_PRIORITY) return; if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) @@ -170,6 +178,8 @@ ThreadData::IncreasePenalty() inline void ThreadData::CancelPenalty() { + SCHEDULER_ENTER_FUNCTION(); + if (fPriorityPenalty != 0) { TRACE("cancelling thread %ld penalty\n", fThread->id); fEffectivePriority = -1; @@ -183,6 +193,8 @@ ThreadData::CancelPenalty() inline bool ThreadData::ShouldCancelPenalty() const { + SCHEDULER_ENTER_FUNCTION(); + if (fCore == NULL) return false; @@ -194,6 +206,7 @@ ThreadData::ShouldCancelPenalty() const inline void ThreadData::IncreaseStolenTime(bigtime_t stolenTime) { + SCHEDULER_ENTER_FUNCTION(); fStolenTime += stolenTime; } @@ -201,6 +214,8 @@ ThreadData::IncreaseStolenTime(bigtime_t stolenTime) inline void ThreadData::GoesAway() { + SCHEDULER_ENTER_FUNCTION(); + fLastInterruptTime = 0; fWentSleep = system_time(); @@ -212,6 +227,8 @@ ThreadData::GoesAway() inline void ThreadData::PutBack() { + SCHEDULER_ENTER_FUNCTION(); + ComputeLoad(); fWentSleepCount = -1; @@ -234,6 +251,8 @@ ThreadData::PutBack() inline void ThreadData::Enqueue() { + SCHEDULER_ENTER_FUNCTION(); + fThread->state = B_THREAD_READY; ComputeLoad(); fWentSleepCount = 0; @@ -256,6 +275,8 @@ ThreadData::Enqueue() inline bool ThreadData::Dequeue() { + SCHEDULER_ENTER_FUNCTION(); + CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; @@ -278,6 +299,8 @@ ThreadData::Dequeue() inline void ThreadData::UpdateActivity(bigtime_t active) { + SCHEDULER_ENTER_FUNCTION(); + fMeasureActiveTime += active; CPUEntry::GetCPU(smp_get_current_cpu())->IncreaseActiveTime(active); fCore->IncreaseActiveTime(active); @@ -287,6 +310,8 @@ ThreadData::UpdateActivity(bigtime_t active) inline void ThreadData::ComputeLoad() { + SCHEDULER_ENTER_FUNCTION(); + if (fLastInterruptTime > 0) { bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; interruptTime -= fLastInterruptTime; @@ -300,6 +325,8 @@ ThreadData::ComputeLoad() inline bool ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) { + SCHEDULER_ENTER_FUNCTION(); + if (hasYielded) { fTimeLeft = 0; return true; @@ -322,6 +349,7 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) inline void ThreadData::StartQuantum() { + SCHEDULER_ENTER_FUNCTION(); fQuantumStart = system_time(); } @@ -329,6 +357,8 @@ ThreadData::StartQuantum() inline int32 ThreadData::_GetPenalty() const { + SCHEDULER_ENTER_FUNCTION(); + int32 penalty = fPriorityPenalty; const int kMinimalPriority = _GetMinimalPriority(); @@ -342,6 +372,8 @@ ThreadData::_GetPenalty() const inline int32 ThreadData::_GetMinimalPriority() const { + SCHEDULER_ENTER_FUNCTION(); + const int32 kDivisor = 5; const int32 kMaximalPriority = 25; From 9a6868d5657e17b8d57896888f5198c5ab6c4bd4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 26 Dec 2013 19:55:33 +0100 Subject: [PATCH 227/273] x86: Fix build with kernel debugging disabled --- src/system/kernel/arch/x86/arch_smp.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index bb95501f38..e9d4b4c545 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include From 335c60552c275dc13e1ca4fac0af2cd11be7f4aa Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 26 Dec 2013 19:56:50 +0100 Subject: [PATCH 228/273] scheduler: Remove CPUEntry::IncreaseActiveTime() --- src/system/kernel/scheduler/scheduler_cpu.cpp | 5 ++++- src/system/kernel/scheduler/scheduler_cpu.h | 11 ----------- src/system/kernel/scheduler/scheduler_thread.h | 3 --- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index f2030759f9..ef6c9bc75b 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -242,6 +242,9 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) cpuEntry->active_time += active; locker.Unlock(); + fMeasureActiveTime += active; + fCore->IncreaseActiveTime(active); + oldThreadData->UpdateActivity(active); } @@ -268,7 +271,7 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) SCHEDULER_ENTER_FUNCTION(); int32 load = std::max(threadData->GetLoad(), fCore->GetLoad()); - load = std::min(std::max(load, int32(0)), kMaxLoad); + ASSERT(load >= 0 && load <= kMaxLoad); if (load < kTargetLoad) { int32 delta = kTargetLoad - load; diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 859627972b..9115f67fbd 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -69,9 +69,6 @@ public: void UpdatePriority(int32 priority); - inline void IncreaseActiveTime( - bigtime_t activeTime); - inline int32 GetLoad() const { return fLoad; } void ComputeLoad(); @@ -307,14 +304,6 @@ CPUEntry::UnlockScheduler() } -inline void -CPUEntry::IncreaseActiveTime(bigtime_t activeTime) -{ - SCHEDULER_ENTER_FUNCTION(); - fMeasureActiveTime += activeTime; -} - - /* static */ inline CPUEntry* CPUEntry::GetCPU(int32 cpu) { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 3fb2db65e2..38434e4122 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -300,10 +300,7 @@ inline void ThreadData::UpdateActivity(bigtime_t active) { SCHEDULER_ENTER_FUNCTION(); - fMeasureActiveTime += active; - CPUEntry::GetCPU(smp_get_current_cpu())->IncreaseActiveTime(active); - fCore->IncreaseActiveTime(active); } From 65d28952ae2284ba71a3396ee11993533f4aacbf Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 26 Dec 2013 22:07:26 +0100 Subject: [PATCH 229/273] scheduler: Make sure RunQueueLink::{fPrevious, fNext} are valid --- src/system/kernel/scheduler/RunQueue.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index 9d54a3ad79..b0ee6ca12d 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -106,7 +106,6 @@ private: }; -#if KDEBUG template RunQueueLink::RunQueueLink() : @@ -114,12 +113,6 @@ RunQueueLink::RunQueueLink() fNext(NULL) { } -#else -template -RunQueueLink::RunQueueLink() -{ -} -#endif template @@ -349,10 +342,8 @@ RUN_QUEUE_CLASS_NAME::Remove(Element* element) if (fHeads[priority] == NULL) fBitmap.Clear(priority); -#if KDEBUG elementLink->fPrevious = NULL; elementLink->fNext = NULL; -#endif } From f68a486001635e28f4542780bfeb685cfa4371ad Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 28 Dec 2013 18:55:39 +0100 Subject: [PATCH 230/273] scheduler: Ignore time used by the profiler implementation --- .../kernel/scheduler/scheduler_profiler.cpp | 18 ++++++++++++++++-- .../kernel/scheduler/scheduler_profiler.h | 1 + 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_profiler.cpp b/src/system/kernel/scheduler/scheduler_profiler.cpp index e0746c1838..64759f52f7 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.cpp +++ b/src/system/kernel/scheduler/scheduler_profiler.cpp @@ -55,6 +55,8 @@ Profiler::Profiler() void Profiler::EnterFunction(int32 cpu, const char* functionName) { + bigtime_t start = system_time(); + FunctionData* function = _FindFunction(functionName); if (function == NULL) return; @@ -67,27 +69,39 @@ Profiler::EnterFunction(int32 cpu, const char* functionName) ASSERT(fFunctionStackPointers[cpu] < kMaxFunctionStackEntries); stackEntry->fFunction = function; - stackEntry->fEntryTime = system_time(); + stackEntry->fEntryTime = start; stackEntry->fOthersTime = 0; + + bigtime_t stop = system_time(); + stackEntry->fProfilerTime = stop - start; } void Profiler::ExitFunction(int32 cpu, const char* functionName) { + bigtime_t start = system_time(); + ASSERT(fFunctionStackPointers[cpu] > 0); fFunctionStackPointers[cpu]--; FunctionEntry* stackEntry = &fFunctionStacks[cpu][fFunctionStackPointers[cpu]]; - bigtime_t timeSpent = system_time() - stackEntry->fEntryTime; + bigtime_t timeSpent = start - stackEntry->fEntryTime; + timeSpent -= stackEntry->fProfilerTime; + atomic_add64(&stackEntry->fFunction->fTimeInclusive, timeSpent); atomic_add64(&stackEntry->fFunction->fTimeExclusive, timeSpent - stackEntry->fOthersTime); + bigtime_t profilerTime = stackEntry->fProfilerTime; if (fFunctionStackPointers[cpu] > 0) { stackEntry = &fFunctionStacks[cpu][fFunctionStackPointers[cpu] - 1]; stackEntry->fOthersTime += timeSpent; + stackEntry->fProfilerTime += profilerTime; + + bigtime_t stop = system_time(); + stackEntry->fProfilerTime += stop - start; } } diff --git a/src/system/kernel/scheduler/scheduler_profiler.h b/src/system/kernel/scheduler/scheduler_profiler.h index fb87c160a8..2b999827a8 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.h +++ b/src/system/kernel/scheduler/scheduler_profiler.h @@ -57,6 +57,7 @@ private: bigtime_t fEntryTime; bigtime_t fOthersTime; + bigtime_t fProfilerTime; }; uint32 _FunctionCount() const; From 4c25fcab386a4a5e22c869865a9b5f25350c24dd Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 28 Dec 2013 20:11:30 +0100 Subject: [PATCH 231/273] scheduler: Fix double release of run queue lock --- src/system/kernel/scheduler/scheduler_thread.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 38434e4122..7bea75bf41 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -244,7 +244,6 @@ ThreadData::PutBack() cpu->PushFront(this, priority); } else fCore->PushFront(this, priority); - fCore->UnlockRunQueue(); } From ca9137de378f56f7a3ef4e0274f2cc1e7d6b0132 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 28 Dec 2013 20:39:42 +0100 Subject: [PATCH 232/273] scheduler: Profile RunQueue implementation --- src/system/kernel/scheduler/RunQueue.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index b0ee6ca12d..6ad9fc444c 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -11,6 +11,8 @@ #include +#include "scheduler_profiler.h" + template struct RunQueueLink { @@ -245,6 +247,8 @@ RUN_QUEUE_TEMPLATE_LIST Element* RUN_QUEUE_CLASS_NAME::PeekMaximum() const { + SCHEDULER_ENTER_FUNCTION(); + int priority = fBitmap.GetHighestSet(); if (priority < 0) return NULL; @@ -268,6 +272,8 @@ void RUN_QUEUE_CLASS_NAME::PushFront(Element* element, unsigned int priority) { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(priority <= MaxPriority); RunQueueLink* elementLink = sGetLink(element); @@ -295,6 +301,8 @@ void RUN_QUEUE_CLASS_NAME::PushBack(Element* element, unsigned int priority) { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(priority <= MaxPriority); RunQueueLink* elementLink = sGetLink(element); @@ -321,6 +329,8 @@ RUN_QUEUE_TEMPLATE_LIST void RUN_QUEUE_CLASS_NAME::Remove(Element* element) { + SCHEDULER_ENTER_FUNCTION(); + RunQueueLink* elementLink = sGetLink(element); unsigned int priority = elementLink->fPriority; @@ -351,6 +361,8 @@ RUN_QUEUE_TEMPLATE_LIST Element* RUN_QUEUE_CLASS_NAME::GetHead(unsigned int priority) const { + SCHEDULER_ENTER_FUNCTION(); + ASSERT(priority <= MaxPriority); return fHeads[priority]; } From 712f37e19e2d0e75c7cc28ffc226b3a62b955ada Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 28 Dec 2013 21:03:43 +0100 Subject: [PATCH 233/273] scheduler: profiler: Use nanosecond as a main time unit --- .../kernel/scheduler/scheduler_profiler.cpp | 20 +++++++++---------- .../kernel/scheduler/scheduler_profiler.h | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_profiler.cpp b/src/system/kernel/scheduler/scheduler_profiler.cpp index 64759f52f7..12a2042b24 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.cpp +++ b/src/system/kernel/scheduler/scheduler_profiler.cpp @@ -55,7 +55,7 @@ Profiler::Profiler() void Profiler::EnterFunction(int32 cpu, const char* functionName) { - bigtime_t start = system_time(); + nanotime_t start = system_time_nsecs(); FunctionData* function = _FindFunction(functionName); if (function == NULL) @@ -72,7 +72,7 @@ Profiler::EnterFunction(int32 cpu, const char* functionName) stackEntry->fEntryTime = start; stackEntry->fOthersTime = 0; - bigtime_t stop = system_time(); + nanotime_t stop = system_time_nsecs(); stackEntry->fProfilerTime = stop - start; } @@ -80,27 +80,27 @@ Profiler::EnterFunction(int32 cpu, const char* functionName) void Profiler::ExitFunction(int32 cpu, const char* functionName) { - bigtime_t start = system_time(); + nanotime_t start = system_time_nsecs(); ASSERT(fFunctionStackPointers[cpu] > 0); fFunctionStackPointers[cpu]--; FunctionEntry* stackEntry = &fFunctionStacks[cpu][fFunctionStackPointers[cpu]]; - bigtime_t timeSpent = start - stackEntry->fEntryTime; + nanotime_t timeSpent = start - stackEntry->fEntryTime; timeSpent -= stackEntry->fProfilerTime; atomic_add64(&stackEntry->fFunction->fTimeInclusive, timeSpent); atomic_add64(&stackEntry->fFunction->fTimeExclusive, timeSpent - stackEntry->fOthersTime); - bigtime_t profilerTime = stackEntry->fProfilerTime; + nanotime_t profilerTime = stackEntry->fProfilerTime; if (fFunctionStackPointers[cpu] > 0) { stackEntry = &fFunctionStacks[cpu][fFunctionStackPointers[cpu] - 1]; stackEntry->fOthersTime += timeSpent; stackEntry->fProfilerTime += profilerTime; - bigtime_t stop = system_time(); + nanotime_t stop = system_time_nsecs(); stackEntry->fProfilerTime += stop - start; } } @@ -126,7 +126,7 @@ Profiler::DumpTimeInclusive(uint32 maxCount) uint32 count = _FunctionCount(); qsort(fFunctionData, count, sizeof(FunctionData), - &_CompareFunctions); + &_CompareFunctions); if (maxCount > 0) count = std::min(count, maxCount); @@ -140,7 +140,7 @@ Profiler::DumpTimeExclusive(uint32 maxCount) uint32 count = _FunctionCount(); qsort(fFunctionData, count, sizeof(FunctionData), - &_CompareFunctions); + &_CompareFunctions); if (maxCount > 0) count = std::min(count, maxCount); @@ -154,7 +154,7 @@ Profiler::DumpTimeInclusivePerCall(uint32 maxCount) uint32 count = _FunctionCount(); qsort(fFunctionData, count, sizeof(FunctionData), - &_CompareFunctionsPerCall); + &_CompareFunctionsPerCall); if (maxCount > 0) count = std::min(count, maxCount); @@ -168,7 +168,7 @@ Profiler::DumpTimeExclusivePerCall(uint32 maxCount) uint32 count = _FunctionCount(); qsort(fFunctionData, count, sizeof(FunctionData), - &_CompareFunctionsPerCall); + &_CompareFunctionsPerCall); if (maxCount > 0) count = std::min(count, maxCount); diff --git a/src/system/kernel/scheduler/scheduler_profiler.h b/src/system/kernel/scheduler/scheduler_profiler.h index 2b999827a8..d2c1bfca3d 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.h +++ b/src/system/kernel/scheduler/scheduler_profiler.h @@ -55,9 +55,9 @@ private: struct FunctionEntry { FunctionData* fFunction; - bigtime_t fEntryTime; - bigtime_t fOthersTime; - bigtime_t fProfilerTime; + nanotime_t fEntryTime; + nanotime_t fOthersTime; + nanotime_t fProfilerTime; }; uint32 _FunctionCount() const; From ef8e55a1d09185c714afac7b5d00f28064af3428 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sat, 28 Dec 2013 21:47:35 +0100 Subject: [PATCH 234/273] scheduler: Use single ended heap for CPU heap --- src/system/kernel/scheduler/low_latency.cpp | 2 +- src/system/kernel/scheduler/power_saving.cpp | 4 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 68 ++++++++++++------- src/system/kernel/scheduler/scheduler_cpu.h | 9 ++- .../kernel/scheduler/scheduler_thread.cpp | 2 +- 5 files changed, 55 insertions(+), 30 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index f76673be88..1e0c79d267 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -138,7 +138,7 @@ rebalance_irqs(bool idle) other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); - int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); + int32 newCPU = other->CPUHeap()->PeekRoot()->ID(); ASSERT(other != NULL); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index c1da765748..4239fd150b 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -174,7 +174,7 @@ pack_irqs() irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID(); + int32 newCPU = smallTaskCore->CPUHeap()->PeekRoot()->ID(); if (newCPU != cpu->cpu_num) assign_io_interrupt_to_cpu(irq->irq, newCPU); @@ -219,7 +219,7 @@ rebalance_irqs(bool idle) coreLocker.Unlock(); if (other == NULL) return; - int32 newCPU = other->CPUHeap()->PeekMinimum()->ID(); + int32 newCPU = other->CPUHeap()->PeekRoot()->ID(); CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); if (other == core) diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index ef6c9bc75b..753f9b30bd 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -146,22 +146,15 @@ CPUEntry::UpdatePriority(int32 priority) if (gCPU[fCPUNumber].disabled) return; - CPUPriorityHeap* cpuHeap = fCore->CPUHeap(); - int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); - cpuHeap->ModifyKey(this, priority); - - if (gSingleCore) + int32 oldPriority = CPUPriorityHeap::GetKey(this); + if (oldPriority == priority) return; + fCore->CPUHeap()->ModifyKey(this, priority); - int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); - if (corePriority == maxPriority) - return; - - PackageEntry* packageEntry = fCore->Package(); - if (maxPriority == B_IDLE_PRIORITY) - packageEntry->CoreGoesIdle(fCore); - else if (corePriority == B_IDLE_PRIORITY) - packageEntry->CoreWakesUp(fCore); + if (oldPriority == B_IDLE_PRIORITY) + fCore->CPUWakesUp(this); + else if (priority == B_IDLE_PRIORITY) + fCore->CPUGoesIdle(this); } @@ -292,7 +285,7 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) CPUPriorityHeap::CPUPriorityHeap(int32 cpuCount) : - MinMaxHeap(cpuCount) + Heap(cpuCount) { } @@ -301,25 +294,25 @@ void CPUPriorityHeap::Dump() { kprintf("cpu priority load\n"); - CPUEntry* entry = PeekMinimum(); + CPUEntry* entry = PeekRoot(); while (entry) { int32 cpu = entry->ID(); int32 key = GetKey(entry); kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, entry->GetLoad() / 10); - RemoveMinimum(); + RemoveRoot(); sDebugCPUHeap.Insert(entry, key); - entry = PeekMinimum(); + entry = PeekRoot(); } - entry = sDebugCPUHeap.PeekMinimum(); + entry = sDebugCPUHeap.PeekRoot(); while (entry) { int32 key = GetKey(entry); - sDebugCPUHeap.RemoveMinimum(); + sDebugCPUHeap.RemoveRoot(); Insert(entry, key); - entry = sDebugCPUHeap.PeekMinimum(); + entry = sDebugCPUHeap.PeekRoot(); } } @@ -327,6 +320,7 @@ CPUPriorityHeap::Dump() CoreEntry::CoreEntry() : fCPUCount(0), + fCPUIdleCount(0), fStarvationCounter(0), fThreadCount(0), fActiveTime(0), @@ -449,10 +443,33 @@ CoreEntry::UpdateLoad(int32 delta) } +inline void +CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) +{ + ASSERT(fCPUIdleCount < fCPUCount); + + if (++fCPUIdleCount == fCPUCount) + fPackage->CoreGoesIdle(this); +} + + +inline void +CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) +{ + ASSERT(fCPUIdleCount > 0); + + if (fCPUIdleCount-- == fCPUCount) + fPackage->CoreWakesUp(this); +} + + void CoreEntry::AddCPU(CPUEntry* cpu) { ASSERT(fCPUCount >= 0); + ASSERT(fCPUIdleCount >= 0); + + fCPUIdleCount++; if (fCPUCount++ == 0) { // core has been reenabled fLoad = 0; @@ -470,6 +487,9 @@ void CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) { ASSERT(fCPUCount > 0); + ASSERT(fCPUIdleCount > 0); + + fCPUIdleCount--; if (--fCPUCount == 0) { // core has been disabled if (fHighLoad) { @@ -499,9 +519,9 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) fThreadCount = 0; } - fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); - ASSERT(fCPUHeap.PeekMaximum() == cpu); - fCPUHeap.RemoveMaximum(); + fCPUHeap.ModifyKey(cpu, -1); + ASSERT(fCPUHeap.PeekRoot() == cpu); + fCPUHeap.RemoveRoot(); ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad); fLoad -= cpu->GetLoad(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 9115f67fbd..2ea2774068 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -41,7 +42,7 @@ public: void Dump() const; }; -class CPUEntry : public MinMaxHeapLinkImpl { +class CPUEntry : public HeapLinkImpl { public: CPUEntry(); @@ -99,7 +100,7 @@ private: friend class DebugDumper; } CACHE_LINE_ALIGN; -class CPUPriorityHeap : public MinMaxHeap { +class CPUPriorityHeap : public Heap { public: CPUPriorityHeap() { } CPUPriorityHeap(int32 cpuCount); @@ -146,6 +147,9 @@ public: inline int32 StarvationCounter() const; + inline void CPUGoesIdle(CPUEntry* cpu); + inline void CPUWakesUp(CPUEntry* cpu); + void AddCPU(CPUEntry* cpu); void RemoveCPU(CPUEntry* cpu, ThreadProcessing& @@ -161,6 +165,7 @@ private: PackageEntry* fPackage; int32 fCPUCount; + int32 fCPUIdleCount; CPUPriorityHeap fCPUHeap; spinlock fCPULock; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index de6a64bb75..4db6516a04 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -205,7 +205,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const } CoreCPUHeapLocker _(core); - CPUEntry* cpu = core->CPUHeap()->PeekMinimum(); + CPUEntry* cpu = core->CPUHeap()->PeekRoot(); ASSERT(cpu != NULL); if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { From 484e5c737f7fdd033c7a83e2f89a28c85ee293d9 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 19:19:21 +0100 Subject: [PATCH 235/273] scheduler: profiler: Compare types larger than int properly --- .../kernel/scheduler/scheduler_profiler.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_profiler.cpp b/src/system/kernel/scheduler/scheduler_profiler.cpp index 12a2042b24..d3399de78b 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.cpp +++ b/src/system/kernel/scheduler/scheduler_profiler.cpp @@ -263,7 +263,11 @@ Profiler::_CompareFunctions(const void* _a, const void* _b) const FunctionData* a = static_cast(_a); const FunctionData* b = static_cast(_b); - return b->*Member - a->*Member; + if (b->*Member > a->*Member) + return 1; + if (b->*Member < a->*Member) + return -1; + return 0; } @@ -274,7 +278,14 @@ Profiler::_CompareFunctionsPerCall(const void* _a, const void* _b) const FunctionData* a = static_cast(_a); const FunctionData* b = static_cast(_b); - return b->*Member / b->fCalled - a->*Member / a->fCalled; + Type valueA = a->*Member / a->fCalled; + Type valueB = b->*Member / b->fCalled; + + if (valueB > valueA) + return 1; + if (valueB < valueA) + return -1; + return 0; } From a47974dd061e73828d099d11c1e4296ee687536b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 19:24:01 +0100 Subject: [PATCH 236/273] scheduler: Use heap to determine highest priority thread --- headers/private/kernel/util/Heap.h | 4 +-- src/system/kernel/scheduler/RunQueue.h | 49 ++++++++++++++++---------- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/headers/private/kernel/util/Heap.h b/headers/private/kernel/util/Heap.h index b2751eb6b3..cecc5a5eb7 100644 --- a/headers/private/kernel/util/Heap.h +++ b/headers/private/kernel/util/Heap.h @@ -78,7 +78,7 @@ public: Heap(int initialSize); ~Heap(); - inline Element* PeekRoot(); + inline Element* PeekRoot() const; static const Key& GetKey(Element* element); @@ -188,7 +188,7 @@ HEAP_CLASS_NAME::~Heap() HEAP_TEMPLATE_LIST Element* -HEAP_CLASS_NAME::PeekRoot() +HEAP_CLASS_NAME::PeekRoot() const { if (fLastElement > 0) return fElements[0]; diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index 6ad9fc444c..1b9a78f38e 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -9,7 +9,7 @@ #define RUN_QUEUE_H -#include +#include #include "scheduler_profiler.h" @@ -97,9 +97,17 @@ public: inline ConstIterator GetConstIterator() const; private: + struct PriorityEntry : public HeapLinkImpl + { + }; + + typedef Heap > + PriorityHeap; + status_t fInitStatus; - Bitmap fBitmap; + PriorityEntry fPriorityEntries[MaxPriority + 1]; + PriorityHeap fPriorityHeap; Element* fHeads[MaxPriority + 1]; Element* fTails[MaxPriority + 1]; @@ -226,10 +234,9 @@ RUN_QUEUE_CLASS_NAME::ConstIterator::_FindNextPriority() RUN_QUEUE_TEMPLATE_LIST RUN_QUEUE_CLASS_NAME::RunQueue() : - fBitmap(MaxPriority + 1) + fInitStatus(B_OK), + fPriorityHeap(MaxPriority + 1) { - fInitStatus = fBitmap.GetInitStatus(); - memset(fHeads, 0, sizeof(fHeads)); memset(fTails, 0, sizeof(fTails)); } @@ -249,17 +256,18 @@ RUN_QUEUE_CLASS_NAME::PeekMaximum() const { SCHEDULER_ENTER_FUNCTION(); - int priority = fBitmap.GetHighestSet(); - if (priority < 0) + PriorityEntry* maxPriority = fPriorityHeap.PeekRoot(); + if (maxPriority == NULL) return NULL; + unsigned int priority = PriorityHeap::GetKey(maxPriority); - ASSERT((unsigned int)priority <= MaxPriority); + ASSERT(priority <= MaxPriority); ASSERT(fHeads[priority] != NULL); Element* element = fHeads[priority]; RunQueueLink* elementLink = sGetLink(element); - ASSERT(elementLink->fPriority == (unsigned int)priority); + ASSERT(elementLink->fPriority == priority); ASSERT(fTails[priority] != NULL); ASSERT(elementLink->fPrevious == NULL); @@ -286,13 +294,13 @@ RUN_QUEUE_CLASS_NAME::PushFront(Element* element, elementLink->fPriority = priority; elementLink->fNext = fHeads[priority]; - if (fHeads[priority]) + if (fHeads[priority] != NULL) sGetLink(fHeads[priority])->fPrevious = element; - else + else { fTails[priority] = element; + fPriorityHeap.Insert(&fPriorityEntries[priority], priority); + } fHeads[priority] = element; - - fBitmap.Set(priority); } @@ -315,13 +323,13 @@ RUN_QUEUE_CLASS_NAME::PushBack(Element* element, elementLink->fPriority = priority; elementLink->fPrevious = fTails[priority]; - if (fTails[priority]) + if (fTails[priority] != NULL) sGetLink(fTails[priority])->fNext = element; - else + else { fHeads[priority] = element; + fPriorityHeap.Insert(&fPriorityEntries[priority], priority); + } fTails[priority] = element; - - fBitmap.Set(priority); } @@ -349,8 +357,11 @@ RUN_QUEUE_CLASS_NAME::Remove(Element* element) ASSERT((fHeads[priority] == NULL && fTails[priority] == NULL) || (fHeads[priority] != NULL && fTails[priority] != NULL)); - if (fHeads[priority] == NULL) - fBitmap.Clear(priority); + if (fHeads[priority] == NULL) { + fPriorityHeap.ModifyKey(&fPriorityEntries[priority], MaxPriority + 1); + ASSERT(fPriorityHeap.PeekRoot() == &fPriorityEntries[priority]); + fPriorityHeap.RemoveRoot(); + } elementLink->fPrevious = NULL; elementLink->fNext = NULL; From 56c0f467b028beb8b5104dd16f51373b20fad9e4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 20:03:59 +0100 Subject: [PATCH 237/273] scheduler: Fix unused variable warnings --- src/system/kernel/scheduler/RunQueue.h | 5 ++--- src/system/kernel/scheduler/scheduler.cpp | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/system/kernel/scheduler/RunQueue.h b/src/system/kernel/scheduler/RunQueue.h index 1b9a78f38e..aae6482195 100644 --- a/src/system/kernel/scheduler/RunQueue.h +++ b/src/system/kernel/scheduler/RunQueue.h @@ -265,11 +265,10 @@ RUN_QUEUE_CLASS_NAME::PeekMaximum() const ASSERT(fHeads[priority] != NULL); Element* element = fHeads[priority]; - RunQueueLink* elementLink = sGetLink(element); - ASSERT(elementLink->fPriority == priority); + ASSERT(sGetLink(element)->fPriority == priority); ASSERT(fTails[priority] != NULL); - ASSERT(elementLink->fPrevious == NULL); + ASSERT(sGetLink(element)->fPrevious == NULL); return element; } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 824dbe4ace..541f5ea9d7 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -700,8 +700,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) CPUEntry* cpu = &gCPUEntries[cpuID]; CoreEntry* core = cpu->Core(); - int32 oldCPUCount = core->CPUCount(); - ASSERT(oldCPUCount >= 0); + ASSERT(core->CPUCount() >= 0); if (enabled) cpu->Start(); else { From e46f284adc620394087b1761e2581bb70f3c9315 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 20:04:36 +0100 Subject: [PATCH 238/273] kernel/util: MinMaxHeap: Fix unused variable warnings --- headers/private/kernel/util/MinMaxHeap.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/headers/private/kernel/util/MinMaxHeap.h b/headers/private/kernel/util/MinMaxHeap.h index 40fd1b1c4c..3eb47e260f 100644 --- a/headers/private/kernel/util/MinMaxHeap.h +++ b/headers/private/kernel/util/MinMaxHeap.h @@ -418,14 +418,13 @@ MIN_MAX_HEAP_TEMPLATE_LIST bool MIN_MAX_HEAP_CLASS_NAME::_ChangeTree(MinMaxHeapLink* link) { - int currentLastElement = link->fMinTree ? fMinLastElement : fMaxLastElement; int otherLastElement = link->fMinTree ? fMaxLastElement : fMinLastElement; Element** currentTree = link->fMinTree ? fMinElements : fMaxElements; Element** otherTree = link->fMinTree ? fMaxElements : fMinElements; if (otherLastElement <= 0) { - ASSERT(currentLastElement == 1); + ASSERT(link->fMinTree ? fMinLastElement : fMaxLastElement == 1); return false; } From c69bd82c3141b437085ad4b9d68427ab91d61ab4 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 22:24:53 +0100 Subject: [PATCH 239/273] apps: ProcessController: Initialize per CPU fields properly --- src/apps/processcontroller/ProcessController.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/apps/processcontroller/ProcessController.cpp b/src/apps/processcontroller/ProcessController.cpp index 0e4a0d08b5..8204e33bc6 100644 --- a/src/apps/processcontroller/ProcessController.cpp +++ b/src/apps/processcontroller/ProcessController.cpp @@ -230,6 +230,10 @@ ProcessController::~ProcessController() void ProcessController::Init() { + memset(fLastBarHeight, 0, sizeof(float) * kCPUCount); + memset(fCPUTimes, 0, sizeof(double) * kCPUCount); + memset(fPrevActive, 0, sizeof(bigtime_t) * kCPUCount); + gPCView = this; fMessageRunner = NULL; memset(fLastBarHeight, 0, sizeof(fLastBarHeight)); From ba4cf03951c0340d761c757a791e4e4540fa911a Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 22:25:37 +0100 Subject: [PATCH 240/273] apps: Pulse: Initialize per CPU fields properly --- src/apps/pulse/PulseView.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/apps/pulse/PulseView.cpp b/src/apps/pulse/PulseView.cpp index 5a31421aac..b4ebb8d9a6 100644 --- a/src/apps/pulse/PulseView.cpp +++ b/src/apps/pulse/PulseView.cpp @@ -62,6 +62,9 @@ PulseView::PulseView(BMessage *message) } void PulseView::Init() { + memset(cpu_times, 0, sizeof(double) * kCPUCount); + memset(prev_active, 0, sizeof(double) * kCPUCount); + popupmenu = new BPopUpMenu("PopUpMenu", false, false, B_ITEMS_IN_COLUMN); popupmenu->SetFont(be_plain_font); mode1 = new BMenuItem("", NULL, 0, 0); From 15a7f2046addd659e66d7614ac1e698a0aade255 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 29 Dec 2013 22:43:01 +0100 Subject: [PATCH 241/273] kernel: Return CPU load in cpu_info --- headers/private/kernel/kscheduler.h | 2 ++ src/system/kernel/scheduler/scheduler.cpp | 7 +++++++ src/system/kernel/system_info.cpp | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index bfc6e0203d..b3328a6329 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -81,6 +81,8 @@ void scheduler_new_thread_entry(Thread* thread); void scheduler_set_cpu_enabled(int32 cpu, bool enabled); +int scheduler_get_cpu_load(int32 cpu); + void scheduler_add_listener(struct SchedulerListener* listener); void scheduler_remove_listener(struct SchedulerListener* listener); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 541f5ea9d7..f8d8f802a9 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -724,6 +724,13 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) } +int +scheduler_get_cpu_load(int32 cpu) +{ + return CPUEntry::GetCPU(cpu)->GetLoad() / 10; +} + + static void traverse_topology_tree(const cpu_topology_node* node, int packageID, int coreID) { diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index 7f7287c232..9f22a905f4 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -452,7 +452,7 @@ get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info) memset(info, 0, sizeof(cpu_info) * count); for (uint32 i = 0; i < count; i++) { info[i].active_time = cpu_get_active_time(firstCPU + i); - // TODO: cpu_info::load + info[i].load = scheduler_get_cpu_load(firstCPU + i); info[i].enabled = !gCPU[firstCPU + i].disabled; } From 8aa1539bb0adeb8442dba8c0fac02534c034c445 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 30 Dec 2013 01:19:14 +0100 Subject: [PATCH 242/273] scheduler: Strengthen CoreEntry::GetLoad() assertion --- src/system/kernel/scheduler/scheduler_cpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 2ea2774068..92de5d1daa 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -386,7 +386,7 @@ CoreEntry::GetLoad() const { SCHEDULER_ENTER_FUNCTION(); - ASSERT(fCPUCount >= 0); + ASSERT(fCPUCount > 0); return fLoad / fCPUCount; } From 046af755d0baf974fb05cb7c017d59413de6a333 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 30 Dec 2013 04:03:28 +0100 Subject: [PATCH 243/273] x86: Fix stack corruption in cache topology detection --- src/system/kernel/arch/x86/arch_cpu.cpp | 26 +++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index f4cc5ffba3..3de6d53835 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -15,6 +15,8 @@ #include #include +#include + #include #include @@ -578,7 +580,7 @@ detect_amd_cache_topology(uint32 maxExtendedLeaf) return; uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; - uint8 maxCacheLevel = 0; + int maxCacheLevel = 0; int currentLevel = 0; int cacheType; @@ -587,17 +589,17 @@ detect_amd_cache_topology(uint32 maxExtendedLeaf) get_current_cpuid(&cpuid, 0x8000001d, currentLevel); cacheType = cpuid.regs.eax & 0x1f; - int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; + if (cacheType == 0) + break; + int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); hierarchyLevels[cacheLevel - 1] = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1); - - if (cacheType != 0) - maxCacheLevel = max_c(maxCacheLevel, cacheLevel); + maxCacheLevel = std::max(maxCacheLevel, cacheLevel); currentLevel++; - } while (cacheType != 0); + } while (true); for (int i = 0; i < maxCacheLevel; i++) sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); @@ -692,7 +694,7 @@ detect_intel_cache_topology(uint32 maxBasicLeaf) return; uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL]; - uint8 maxCacheLevel = 0; + int maxCacheLevel = 0; int currentLevel = 0; int cacheType; @@ -701,16 +703,16 @@ detect_intel_cache_topology(uint32 maxBasicLeaf) get_current_cpuid(&cpuid, 4, currentLevel); cacheType = cpuid.regs.eax & 0x1f; - int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; + if (cacheType == 0) + break; + int cacheLevel = (cpuid.regs.eax >> 5) & 0x7; hierarchyLevels[cacheLevel - 1] = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1); - - if (cacheType != 0) - maxCacheLevel = max_c(maxCacheLevel, cacheLevel); + maxCacheLevel = std::max(maxCacheLevel, cacheLevel); currentLevel++; - } while (cacheType != 0); + } while (true); for (int i = 0; i < maxCacheLevel; i++) sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1); From 1524fbf74231cf23490a81cb08cea0a478e5a524 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 30 Dec 2013 06:07:51 +0100 Subject: [PATCH 244/273] scheduler: Fix divide error in _RequestPerformanceLevel --- src/system/kernel/scheduler/scheduler_cpu.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 753f9b30bd..f1a5b5e377 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -263,6 +263,11 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) { SCHEDULER_ENTER_FUNCTION(); + if (gCPU[fCPUNumber].disabled) { + decrease_cpu_performance(kCPUPerformanceScaleMax); + return; + } + int32 load = std::max(threadData->GetLoad(), fCore->GetLoad()); ASSERT(load >= 0 && load <= kMaxLoad); From 265927509dc56e82b12cd68750ae1e96601fd558 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 30 Dec 2013 20:03:36 +0100 Subject: [PATCH 245/273] scheduler: Protect per CPU run queue with its own lock --- src/system/kernel/scheduler/scheduler.cpp | 2 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 22 ++++--- src/system/kernel/scheduler/scheduler_cpu.h | 36 ++++++++++++ .../kernel/scheduler/scheduler_thread.h | 57 ++++++++++++------- 4 files changed, 86 insertions(+), 31 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index f8d8f802a9..fc941b34fd 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -517,7 +517,7 @@ reschedule(int32 nextState) ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { if (!thread_is_idle_thread(oldThread)) { - CoreRunQueueLocker _(core); + CPURunQueueLocker _(cpu); nextThreadData = cpu->PeekIdleThread(); cpu->Remove(nextThreadData); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index f1a5b5e377..08987fd49a 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -56,6 +56,7 @@ CPUEntry::CPUEntry() fMeasureTime(0) { B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); + B_INITIALIZE_SPINLOCK(&fQueueLock); } @@ -185,25 +186,26 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) { SCHEDULER_ENTER_FUNCTION(); - CoreRunQueueLocker _(fCore); + int32 oldPriority = -1; + if (oldThread != NULL) + oldPriority = oldThread->GetEffectivePriority(); + + CPURunQueueLocker cpuLocker(this); - ThreadData* sharedThread = fCore->PeekThread(); ThreadData* pinnedThread = fRunQueue.PeekMaximum(); - - ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); - int32 pinnedPriority = -1; if (pinnedThread != NULL) pinnedPriority = pinnedThread->GetEffectivePriority(); + CoreRunQueueLocker coreLocker(fCore); + + ThreadData* sharedThread = fCore->PeekThread(); + ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); + int32 sharedPriority = -1; if (sharedThread != NULL) sharedPriority = sharedThread->GetEffectivePriority(); - int32 oldPriority = -1; - if (oldThread != NULL) - oldPriority = oldThread->GetEffectivePriority(); - int32 rest = std::max(pinnedPriority, sharedPriority); if (oldPriority > rest || (!putAtBack && oldPriority == rest)) return oldThread; @@ -213,6 +215,8 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) return sharedThread; } + coreLocker.Unlock(); + Remove(pinnedThread); return pinnedThread; } diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 92de5d1daa..bd152644e7 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -60,6 +60,9 @@ public: inline void LockScheduler(); inline void UnlockScheduler(); + inline void LockRunQueue(); + inline void UnlockRunQueue(); + void PushFront(ThreadData* thread, int32 priority); void PushBack(ThreadData* thread, @@ -91,6 +94,7 @@ private: rw_spinlock fSchedulerModeLock; ThreadRunQueue fRunQueue; + spinlock fQueueLock; int32 fLoad; @@ -108,6 +112,22 @@ public: void Dump(); }; +class CPURunQueueLocking { +public: + inline bool Lock(CPUEntry* cpu) + { + cpu->LockRunQueue(); + return true; + } + + inline void Unlock(CPUEntry* cpu) + { + cpu->UnlockRunQueue(); + } +}; + +typedef AutoLocker CPURunQueueLocker; + class CoreEntry : public MinMaxHeapLinkImpl, public DoublyLinkedListLinkImpl { public: @@ -309,6 +329,22 @@ CPUEntry::UnlockScheduler() } +inline void +CPUEntry::LockRunQueue() +{ + SCHEDULER_ENTER_FUNCTION(); + acquire_spinlock(&fQueueLock); +} + + +inline void +CPUEntry::UnlockRunQueue() +{ + SCHEDULER_ENTER_FUNCTION(); + release_spinlock(&fQueueLock); +} + + /* static */ inline CPUEntry* CPUEntry::GetCPU(int32 cpu) { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 7bea75bf41..d37d8bf5b2 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -234,16 +234,22 @@ ThreadData::PutBack() int32 priority = GetEffectivePriority(); - CoreRunQueueLocker _(fCore); - ASSERT(!fEnqueued); - fEnqueued = true; if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->cpu != NULL); - CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num); + + CPURunQueueLocker _(cpu); + ASSERT(!fEnqueued); + fEnqueued = true; + cpu->PushFront(this, priority); - } else + } else { + CoreRunQueueLocker _(fCore); + ASSERT(!fEnqueued); + fEnqueued = true; + fCore->PushFront(this, priority); + } } @@ -258,16 +264,22 @@ ThreadData::Enqueue() int32 priority = GetEffectivePriority(); - CoreRunQueueLocker _(fCore); - ASSERT(!fEnqueued); - fEnqueued = true; if (fThread->pinned_to_cpu > 0) { ASSERT(fThread->previous_cpu != NULL); - CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + + CPURunQueueLocker _(cpu); + ASSERT(!fEnqueued); + fEnqueued = true; + cpu->PushBack(this, priority); - } else + } else { + CoreRunQueueLocker _(fCore); + ASSERT(!fEnqueued); + fEnqueued = true; + fCore->PushBack(this, priority); + } } @@ -276,20 +288,23 @@ ThreadData::Dequeue() { SCHEDULER_ENTER_FUNCTION(); + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->previous_cpu != NULL); + CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); + + CPURunQueueLocker _(cpu); + if (!fEnqueued) + return false; + cpu->Remove(this); + ASSERT(!fEnqueued); + return true; + } + CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; - - if (fThread->pinned_to_cpu > 0) { - ASSERT(fThread->previous_cpu != NULL); - - CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); - cpu->Remove(this); - } else { - ASSERT(fWentSleepCount < 1); - fCore->Remove(this); - } - + ASSERT(fWentSleepCount < 1); + fCore->Remove(this); ASSERT(!fEnqueued); return true; } From 7c92dffeef958b137733dbfc75d0f54f9fad1f5e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 30 Dec 2013 20:09:24 +0100 Subject: [PATCH 246/273] scheduler: Fix order of the includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks Jérôme! --- src/system/kernel/scheduler/scheduler_cpu.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index bd152644e7..c76ccf1506 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -10,8 +10,8 @@ #include #include -#include #include +#include #include From 2d52abbd5d279c622982b8cae1f38d51bf73c2d2 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 31 Dec 2013 00:59:29 +0100 Subject: [PATCH 247/273] scheduler: Inherit penalty and core from creator thread --- src/system/kernel/scheduler/low_latency.cpp | 3 ++- src/system/kernel/scheduler/power_saving.cpp | 2 ++ .../kernel/scheduler/scheduler_thread.cpp | 22 ++++++++++++++++--- .../kernel/scheduler/scheduler_thread.h | 2 +- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 1e0c79d267..42baee3325 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -35,7 +35,8 @@ static bool has_cache_expired(const ThreadData* threadData) { SCHEDULER_ENTER_FUNCTION(); - + if (threadData->WentSleepActive() == 0) + return false; CoreEntry* core = threadData->Core(); bigtime_t activeTime = core->GetActiveTime(); return activeTime - threadData->WentSleepActive() > kCacheExpire; diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 4239fd150b..3b36add0c9 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -41,6 +41,8 @@ static bool has_cache_expired(const ThreadData* threadData) { SCHEDULER_ENTER_FUNCTION(); + if (threadData->WentSleep() == 0) + return false; return system_time() - threadData->WentSleep() > kCacheExpire; } diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 4db6516a04..9f5d834d0b 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -16,14 +16,12 @@ ThreadData::ThreadData(Thread* thread) : fThread(thread) { - Init(); } void ThreadData::Init() { - fPriorityPenalty = 0; fAdditionalPenalty = 0; fEffectivePriority = -1; @@ -40,7 +38,25 @@ ThreadData::Init() fEnqueued = false; - fCore = NULL; + Thread* currentThread = thread_get_current_thread(); + ASSERT(currentThread != NULL); + if (!thread_is_idle_thread(currentThread)) { + ThreadData* currentThreadData = currentThread->scheduler_data; + int32 penalty = currentThreadData->fPriorityPenalty; + + int32 minimalPriority = _GetMinimalPriority(); + if (fThread->priority - penalty >= minimalPriority) + fPriorityPenalty = penalty; + else + fPriorityPenalty = fThread->priority - minimalPriority; + + fCore = currentThreadData->fCore; + } else { + fPriorityPenalty = 0; + fAdditionalPenalty = 0; + + fCore = NULL; + } } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index d37d8bf5b2..98cc660cc6 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -195,7 +195,7 @@ ThreadData::ShouldCancelPenalty() const { SCHEDULER_ENTER_FUNCTION(); - if (fCore == NULL) + if (fCore == NULL || fWentSleep == 0) return false; return fCore->StarvationCounter() != fWentSleepCount From 135bb9c9596ff16f1d775ca48f6ae884af7dc52b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 31 Dec 2013 03:50:01 +0100 Subject: [PATCH 248/273] kernel: Do not attempt to interrupt a thread that is not waiting --- headers/private/kernel/thread.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index 96d80ba912..f41987c8c4 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -383,10 +383,12 @@ thread_unblock_locked(Thread* thread, status_t status) static inline status_t thread_interrupt(Thread* thread, bool kill) { - if ((thread->wait.flags & B_CAN_INTERRUPT) != 0 - || (kill && (thread->wait.flags & B_KILL_CAN_INTERRUPT) != 0)) { - thread_unblock_locked(thread, B_INTERRUPTED); - return B_OK; + if (thread_is_blocked(thread)) { + if ((thread->wait.flags & B_CAN_INTERRUPT) != 0 + || (kill && (thread->wait.flags & B_KILL_CAN_INTERRUPT) != 0)) { + thread_unblock_locked(thread, B_INTERRUPTED); + return B_OK; + } } return B_NOT_ALLOWED; From db1ddabfd0121c18ab9d87e2047c6ff6e4d29653 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 31 Dec 2013 04:48:23 +0100 Subject: [PATCH 249/273] scheduler: Thread can not stay on CPU if it has been disabled --- src/system/kernel/scheduler/scheduler.cpp | 8 +++----- src/system/kernel/scheduler/scheduler_thread.cpp | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index fc941b34fd..41546f05da 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -532,6 +532,9 @@ reschedule(int32 nextState) } Thread* nextThread = nextThreadData->GetThread(); + ASSERT(!gCPU[thisCPU].disabled || thread_is_idle_thread(nextThread)); + + // update CPU heap CoreCPUHeapLocker cpuLocker(core); cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); cpuLocker.Unlock(); @@ -559,11 +562,6 @@ reschedule(int32 nextState) ASSERT(nextThreadData->Core() == core); nextThread->state = B_THREAD_RUNNING; - // update CPU heap - cpuLocker.Lock(); - cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); - cpuLocker.Unlock(); - // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 9f5d834d0b..4d5ea1909b 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -210,7 +210,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const if (fThread->previous_cpu != NULL) { CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; - if (previousCPU->Core() == core) { + if (previousCPU->Core() == core && !fThread->previous_cpu->disabled) { CoreCPUHeapLocker _(core); if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { previousCPU->UpdatePriority(threadPriority); From e4ea637227d7cf9a53bc89317990b8a22a76780a Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 3 Jan 2014 19:34:25 +0100 Subject: [PATCH 250/273] scheduler: Disable load tracking when not needed --- src/system/kernel/scheduler/scheduler.cpp | 13 +++++++++++-- src/system/kernel/scheduler/scheduler_common.h | 2 ++ src/system/kernel/scheduler/scheduler_cpu.cpp | 6 ++++-- .../kernel/scheduler/scheduler_thread.cpp | 18 ++++++++++++++++++ src/system/kernel/scheduler/scheduler_thread.h | 17 +---------------- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 41546f05da..bb3caa8034 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -136,6 +136,8 @@ scheduler_mode gCurrentModeID; scheduler_mode_operations* gCurrentMode; bool gSingleCore; +bool gCPUFrequencyManagement; +bool gTrackLoad; CPUEntry* gCPUEntries; @@ -533,7 +535,6 @@ reschedule(int32 nextState) Thread* nextThread = nextThreadData->GetThread(); ASSERT(!gCPU[thisCPU].disabled || thread_is_idle_thread(nextThread)); - // update CPU heap CoreCPUHeapLocker cpuLocker(core); cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); @@ -803,8 +804,16 @@ init() if (result != B_OK) return result; - gCoreCount = coreCount; + // disable parts of the scheduler logic that are not needed gSingleCore = coreCount == 1; + gCPUFrequencyManagement = increase_cpu_performance(0) == B_OK; + gTrackLoad = !gSingleCore || gCPUFrequencyManagement; + dprintf("scheduler switches: single core: %s, cpufreq: %s, load tracking:" + " %s\n", gSingleCore ? "true" : "false", + gCPUFrequencyManagement ? "true" : "false", + gTrackLoad ? "true" : "false"); + + gCoreCount = coreCount; gPackageCount = packageCount; gCPUEntries = new(std::nothrow) CPUEntry[cpuCount]; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 02c96917f2..812fb7d8d0 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -43,6 +43,8 @@ const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; const int kLoadDifference = kMaxLoad * 20 / 100; extern bool gSingleCore; +extern bool gCPUFrequencyManagement; +extern bool gTrackLoad; void init_debug_commands(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 08987fd49a..a88c041693 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -164,7 +164,6 @@ CPUEntry::ComputeLoad() { SCHEDULER_ENTER_FUNCTION(); - ASSERT(!gSingleCore); ASSERT(fCPUNumber == smp_get_current_cpu()); int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad); @@ -247,7 +246,7 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) oldThreadData->ComputeLoad(); nextThreadData->ComputeLoad(); - if (!gSingleCore && !cpuEntry->disabled) + if (gTrackLoad && !cpuEntry->disabled) ComputeLoad(); Thread* nextThread = nextThreadData->GetThread(); @@ -267,6 +266,9 @@ CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) { SCHEDULER_ENTER_FUNCTION(); + if (!gCPUFrequencyManagement) + return; + if (gCPU[fCPUNumber].disabled) { decrease_cpu_performance(kCPUPerformanceScaleMax); return; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 4d5ea1909b..ec84ee46ec 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -115,6 +115,24 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) } +void +ThreadData::ComputeLoad() +{ + SCHEDULER_ENTER_FUNCTION(); + + if (!gTrackLoad) + return; + + if (fLastInterruptTime > 0) { + bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; + interruptTime -= fLastInterruptTime; + fMeasureActiveTime -= interruptTime; + } + + compute_load(fMeasureTime, fMeasureActiveTime, fLoad); +} + + bigtime_t ThreadData::ComputeQuantum() { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 98cc660cc6..debcf2d98b 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -56,7 +56,7 @@ public: inline bool Dequeue(); inline void UpdateActivity(bigtime_t active); - inline void ComputeLoad(); + void ComputeLoad(); inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); bigtime_t ComputeQuantum(); @@ -318,21 +318,6 @@ ThreadData::UpdateActivity(bigtime_t active) } -inline void -ThreadData::ComputeLoad() -{ - SCHEDULER_ENTER_FUNCTION(); - - if (fLastInterruptTime > 0) { - bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; - interruptTime -= fLastInterruptTime; - fMeasureActiveTime -= interruptTime; - } - - compute_load(fMeasureTime, fMeasureActiveTime, fLoad); -} - - inline bool ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) { From 81e04d7b9734081a6c3190b9b8bdab304dc0fe3e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 3 Jan 2014 19:38:05 +0100 Subject: [PATCH 251/273] kernel: Remove cpu_info::load This field forces kernel to track each CPU load all the time. It is not a problem with the current scheduler on a multicore systems, but on single core machnies or with any other future scheduler this field may become just an unnecessary burden. It isn't difficult for an application to compute CPU load by itself when it needs it. --- headers/os/kernel/OS.h | 1 - headers/private/kernel/kscheduler.h | 2 -- src/system/kernel/scheduler/scheduler.cpp | 7 ------- src/system/kernel/system_info.cpp | 1 - 4 files changed, 11 deletions(-) diff --git a/headers/os/kernel/OS.h b/headers/os/kernel/OS.h index 66131ee825..dce55f051e 100644 --- a/headers/os/kernel/OS.h +++ b/headers/os/kernel/OS.h @@ -422,7 +422,6 @@ extern void ktrace_vprintf(const char *format, va_list args); typedef struct { bigtime_t active_time; /* usec of doing useful work since boot */ - int32 load; bool enabled; } cpu_info; diff --git a/headers/private/kernel/kscheduler.h b/headers/private/kernel/kscheduler.h index b3328a6329..bfc6e0203d 100644 --- a/headers/private/kernel/kscheduler.h +++ b/headers/private/kernel/kscheduler.h @@ -81,8 +81,6 @@ void scheduler_new_thread_entry(Thread* thread); void scheduler_set_cpu_enabled(int32 cpu, bool enabled); -int scheduler_get_cpu_load(int32 cpu); - void scheduler_add_listener(struct SchedulerListener* listener); void scheduler_remove_listener(struct SchedulerListener* listener); diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index bb3caa8034..04827d8216 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -723,13 +723,6 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) } -int -scheduler_get_cpu_load(int32 cpu) -{ - return CPUEntry::GetCPU(cpu)->GetLoad() / 10; -} - - static void traverse_topology_tree(const cpu_topology_node* node, int packageID, int coreID) { diff --git a/src/system/kernel/system_info.cpp b/src/system/kernel/system_info.cpp index 9f22a905f4..5fa166436e 100644 --- a/src/system/kernel/system_info.cpp +++ b/src/system/kernel/system_info.cpp @@ -452,7 +452,6 @@ get_cpu_info(uint32 firstCPU, uint32 cpuCount, cpu_info* info) memset(info, 0, sizeof(cpu_info) * count); for (uint32 i = 0; i < count; i++) { info[i].active_time = cpu_get_active_time(firstCPU + i); - info[i].load = scheduler_get_cpu_load(firstCPU + i); info[i].enabled = !gCPU[firstCPU + i].disabled; } From 9e99bf6085a30c0fdb30234af1924007e6033494 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 3 Jan 2014 20:09:43 +0100 Subject: [PATCH 252/273] scheduler: SCHEDULER_EXIT_FUNCTION() is not needed --- src/system/kernel/scheduler/scheduler.cpp | 2 -- src/system/kernel/scheduler/scheduler_profiler.h | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 04827d8216..de6a49fd9e 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -585,8 +585,6 @@ reschedule(int32 nextState) modeLocker.Unlock(); - SCHEDULER_EXIT_FUNCTION(); - if (nextThread != oldThread) switch_thread(oldThread, nextThread); } diff --git a/src/system/kernel/scheduler/scheduler_profiler.h b/src/system/kernel/scheduler/scheduler_profiler.h index d2c1bfca3d..e6c95b5703 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.h +++ b/src/system/kernel/scheduler/scheduler_profiler.h @@ -16,9 +16,6 @@ #define SCHEDULER_ENTER_FUNCTION() \ Scheduler::Profiling::Function schedulerProfiler(__PRETTY_FUNCTION__) -#define SCHEDULER_EXIT_FUNCTION() \ - schedulerProfiler.Exit() - namespace Scheduler { @@ -127,7 +124,6 @@ Function::Exit() #else // SCHEDULER_PROFILING #define SCHEDULER_ENTER_FUNCTION() (void)0 -#define SCHEDULER_EXIT_FUNCTION() (void)0 #endif // !SCHEDULER_PROFILING From cb66faef24f64af40a51f23300ff546d975535b3 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Sun, 5 Jan 2014 05:55:13 +0100 Subject: [PATCH 253/273] scheduler: Work around GCC2 limitations in function inlining GCC2 won't inline a function if it is used before its definition. --- headers/private/kernel/smp.h | 9 +- src/system/kernel/scheduler/scheduler.cpp | 90 +--------- src/system/kernel/scheduler/scheduler_cpu.cpp | 70 -------- src/system/kernel/scheduler/scheduler_cpu.h | 123 ++++++++------ .../kernel/scheduler/scheduler_locking.h | 158 ++++++++++++++++++ .../kernel/scheduler/scheduler_thread.cpp | 119 +++++++------ .../kernel/scheduler/scheduler_thread.h | 65 ++++--- 7 files changed, 341 insertions(+), 293 deletions(-) create mode 100644 src/system/kernel/scheduler/scheduler_locking.h diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 88a89e4064..3378f23d26 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -41,7 +41,7 @@ typedef void (*smp_call_func)(addr_t data1, int32 currentCPU, addr_t data2, addr class CPUSet { public: - CPUSet() { ClearAll(); } + inline CPUSet(); inline void ClearAll(); inline void SetAll(); @@ -92,6 +92,13 @@ int smp_intercpu_int_handler(int32 cpu); #endif +inline +CPUSet::CPUSet() +{ + memset(fBitmap, 0, sizeof(fBitmap)); +} + + inline void CPUSet::ClearAll() { diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index de6a49fd9e..2916b5b9aa 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -31,6 +31,7 @@ #include "scheduler_common.h" #include "scheduler_cpu.h" +#include "scheduler_locking.h" #include "scheduler_modes.h" #include "scheduler_profiler.h" #include "scheduler_thread.h" @@ -40,93 +41,6 @@ namespace Scheduler { -class SchedulerModeLocking { -public: - bool Lock(int* /* lockable */) - { - CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); - return true; - } - - void Unlock(int* /* lockable */) - { - CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); - } -}; - -class SchedulerModeLocker : - public AutoLocker { -public: - SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) - : - AutoLocker(NULL, alreadyLocked, - lockIfNotLocked) - { - } -}; - -class InterruptsSchedulerModeLocking { -public: - bool Lock(int* lockable) - { - *lockable = disable_interrupts(); - CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); - return true; - } - - void Unlock(int* lockable) - { - CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); - restore_interrupts(*lockable); - } -}; - -class InterruptsSchedulerModeLocker : - public AutoLocker { -public: - InterruptsSchedulerModeLocker(bool alreadyLocked = false, - bool lockIfNotLocked = true) - : - AutoLocker(&fState, alreadyLocked, - lockIfNotLocked) - { - } - -private: - int fState; -}; - -class InterruptsBigSchedulerLocking { -public: - bool Lock(int* lockable) - { - *lockable = disable_interrupts(); - for (int32 i = 0; i < smp_get_num_cpus(); i++) - CPUEntry::GetCPU(i)->LockScheduler(); - return true; - } - - void Unlock(int* lockable) - { - for (int32 i = 0; i < smp_get_num_cpus(); i++) - CPUEntry::GetCPU(i)->UnlockScheduler(); - restore_interrupts(*lockable); - } -}; - -class InterruptsBigSchedulerLocker : - public AutoLocker { -public: - InterruptsBigSchedulerLocker() - : - AutoLocker(&fState, false, true) - { - } - -private: - int fState; -}; - class ThreadEnqueuer : public ThreadProcessing { public: void operator()(ThreadData* thread); @@ -624,6 +538,8 @@ scheduler_on_thread_create(Thread* thread, bool idleThread) void scheduler_on_thread_init(Thread* thread) { + ASSERT(thread->scheduler_data != NULL); + if (thread_is_idle_thread(thread)) { static int32 sIdleThreadsID; int32 cpuID = atomic_add(&sIdleThreadsID, 1); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index a88c041693..7ec4ab5eec 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -392,14 +392,6 @@ CoreEntry::Remove(ThreadData* thread) } -inline ThreadData* -CoreEntry::PeekThread() const -{ - SCHEDULER_ENTER_FUNCTION(); - return fRunQueue.PeekMaximum(); -} - - void CoreEntry::UpdateLoad(int32 delta) { @@ -454,26 +446,6 @@ CoreEntry::UpdateLoad(int32 delta) } -inline void -CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) -{ - ASSERT(fCPUIdleCount < fCPUCount); - - if (++fCPUIdleCount == fCPUCount) - fPackage->CoreGoesIdle(this); -} - - -inline void -CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) -{ - ASSERT(fCPUIdleCount > 0); - - if (fCPUIdleCount-- == fCPUCount) - fPackage->CoreWakesUp(this); -} - - void CoreEntry::AddCPU(CPUEntry* cpu) { @@ -599,48 +571,6 @@ PackageEntry::Init(int32 id) } -inline void -PackageEntry::CoreGoesIdle(CoreEntry* core) -{ - SCHEDULER_ENTER_FUNCTION(); - - WriteSpinLocker _(fCoreLock); - - ASSERT(fIdleCoreCount >= 0); - ASSERT(fIdleCoreCount < fCoreCount); - - fIdleCoreCount++; - fIdleCores.Add(core); - - if (fIdleCoreCount == fCoreCount) { - // package goes idle - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList.Add(this); - } -} - - -inline void -PackageEntry::CoreWakesUp(CoreEntry* core) -{ - SCHEDULER_ENTER_FUNCTION(); - - WriteSpinLocker _(fCoreLock); - - ASSERT(fIdleCoreCount > 0); - ASSERT(fIdleCoreCount <= fCoreCount); - - fIdleCoreCount--; - fIdleCores.Remove(core); - - if (fIdleCoreCount + 1 == fCoreCount) { - // package wakes up - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList.Remove(this); - } -} - - void PackageEntry::AddIdleCore(CoreEntry* core) { diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index c76ccf1506..b62ee5f3a2 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -85,7 +85,7 @@ public: static inline CPUEntry* GetCPU(int32 cpu); private: - inline void _RequestPerformanceLevel( + void _RequestPerformanceLevel( ThreadData* threadData); int32 fCPUNumber; @@ -112,21 +112,6 @@ public: void Dump(); }; -class CPURunQueueLocking { -public: - inline bool Lock(CPUEntry* cpu) - { - cpu->LockRunQueue(); - return true; - } - - inline void Unlock(CPUEntry* cpu) - { - cpu->UnlockRunQueue(); - } -}; - -typedef AutoLocker CPURunQueueLocker; class CoreEntry : public MinMaxHeapLinkImpl, public DoublyLinkedListLinkImpl { @@ -205,37 +190,6 @@ private: friend class DebugDumper; } CACHE_LINE_ALIGN; -class CoreRunQueueLocking { -public: - inline bool Lock(CoreEntry* core) - { - core->LockRunQueue(); - return true; - } - - inline void Unlock(CoreEntry* core) - { - core->UnlockRunQueue(); - } -}; - -typedef AutoLocker CoreRunQueueLocker; - -class CoreCPUHeapLocking { -public: - inline bool Lock(CoreEntry* core) - { - core->LockCPUHeap(); - return true; - } - - inline void Unlock(CoreEntry* core) - { - core->UnlockCPUHeap(); - } -}; - -typedef AutoLocker CoreCPUHeapLocker; class CoreLoadHeap : public MinMaxHeap { public: @@ -402,6 +356,14 @@ CoreEntry::IncreaseActiveTime(bigtime_t activeTime) } +inline ThreadData* +CoreEntry::PeekThread() const +{ + SCHEDULER_ENTER_FUNCTION(); + return fRunQueue.PeekMaximum(); +} + + inline bigtime_t CoreEntry::GetActiveTime() const { @@ -435,6 +397,73 @@ CoreEntry::StarvationCounter() const } +/* PackageEntry::CoreGoesIdle and PackageEntry::CoreWakesUp have to be defined + before CoreEntry::CPUGoesIdle and CoreEntry::CPUWakesUp. If they weren't + GCC2 wouldn't inline them as, apparently, it doesn't do enough optimization + passes. +*/ +inline void +PackageEntry::CoreGoesIdle(CoreEntry* core) +{ + SCHEDULER_ENTER_FUNCTION(); + + WriteSpinLocker _(fCoreLock); + + ASSERT(fIdleCoreCount >= 0); + ASSERT(fIdleCoreCount < fCoreCount); + + fIdleCoreCount++; + fIdleCores.Add(core); + + if (fIdleCoreCount == fCoreCount) { + // package goes idle + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Add(this); + } +} + + +inline void +PackageEntry::CoreWakesUp(CoreEntry* core) +{ + SCHEDULER_ENTER_FUNCTION(); + + WriteSpinLocker _(fCoreLock); + + ASSERT(fIdleCoreCount > 0); + ASSERT(fIdleCoreCount <= fCoreCount); + + fIdleCoreCount--; + fIdleCores.Remove(core); + + if (fIdleCoreCount + 1 == fCoreCount) { + // package wakes up + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Remove(this); + } +} + + +inline void +CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) +{ + ASSERT(fCPUIdleCount < fCPUCount); + + if (++fCPUIdleCount == fCPUCount) + fPackage->CoreGoesIdle(this); +} + + +inline void +CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) +{ + ASSERT(fCPUIdleCount > 0); + + if (fCPUIdleCount-- == fCPUCount) + fPackage->CoreWakesUp(this); +} + + /* static */ inline CoreEntry* CoreEntry::GetCore(int32 cpu) { diff --git a/src/system/kernel/scheduler/scheduler_locking.h b/src/system/kernel/scheduler/scheduler_locking.h new file mode 100644 index 0000000000..044015c236 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_locking.h @@ -0,0 +1,158 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_LOCKING_H +#define KERNEL_SCHEDULER_LOCKING_H + + +#include + +#include "scheduler_cpu.h" + + +namespace Scheduler { + + +class CPURunQueueLocking { +public: + inline bool Lock(CPUEntry* cpu) + { + cpu->LockRunQueue(); + return true; + } + + inline void Unlock(CPUEntry* cpu) + { + cpu->UnlockRunQueue(); + } +}; + +typedef AutoLocker CPURunQueueLocker; + + +class CoreRunQueueLocking { +public: + inline bool Lock(CoreEntry* core) + { + core->LockRunQueue(); + return true; + } + + inline void Unlock(CoreEntry* core) + { + core->UnlockRunQueue(); + } +}; + +typedef AutoLocker CoreRunQueueLocker; + +class CoreCPUHeapLocking { +public: + inline bool Lock(CoreEntry* core) + { + core->LockCPUHeap(); + return true; + } + + inline void Unlock(CoreEntry* core) + { + core->UnlockCPUHeap(); + } +}; + +typedef AutoLocker CoreCPUHeapLocker; + +class SchedulerModeLocking { +public: + bool Lock(int* /* lockable */) + { + CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); + return true; + } + + void Unlock(int* /* lockable */) + { + CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); + } +}; + +class SchedulerModeLocker : + public AutoLocker { +public: + SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) + : + AutoLocker(NULL, alreadyLocked, + lockIfNotLocked) + { + } +}; + +class InterruptsSchedulerModeLocking { +public: + bool Lock(int* lockable) + { + *lockable = disable_interrupts(); + CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler(); + return true; + } + + void Unlock(int* lockable) + { + CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler(); + restore_interrupts(*lockable); + } +}; + +class InterruptsSchedulerModeLocker : + public AutoLocker { +public: + InterruptsSchedulerModeLocker(bool alreadyLocked = false, + bool lockIfNotLocked = true) + : + AutoLocker(&fState, alreadyLocked, + lockIfNotLocked) + { + } + +private: + int fState; +}; + +class InterruptsBigSchedulerLocking { +public: + bool Lock(int* lockable) + { + *lockable = disable_interrupts(); + for (int32 i = 0; i < smp_get_num_cpus(); i++) + CPUEntry::GetCPU(i)->LockScheduler(); + return true; + } + + void Unlock(int* lockable) + { + for (int32 i = 0; i < smp_get_num_cpus(); i++) + CPUEntry::GetCPU(i)->UnlockScheduler(); + restore_interrupts(*lockable); + } +}; + +class InterruptsBigSchedulerLocker : + public AutoLocker { +public: + InterruptsBigSchedulerLocker() + : + AutoLocker(&fState, false, true) + { + } + +private: + int fState; +}; + + +} // namespace Scheduler + + +#endif // KERNEL_SCHEDULER_LOCKING_H + diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index ec84ee46ec..aef21a59f1 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -12,6 +12,58 @@ using namespace Scheduler; static bigtime_t sQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; +inline CoreEntry* +ThreadData::_ChooseCore() const +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(!gSingleCore); + return gCurrentMode->choose_core(this); +} + + +inline CPUEntry* +ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const +{ + SCHEDULER_ENTER_FUNCTION(); + + int32 threadPriority = GetEffectivePriority(); + + if (fThread->previous_cpu != NULL) { + CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; + if (previousCPU->Core() == core && !fThread->previous_cpu->disabled) { + CoreCPUHeapLocker _(core); + if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { + previousCPU->UpdatePriority(threadPriority); + rescheduleNeeded = true; + return previousCPU; + } + } + } + + CoreCPUHeapLocker _(core); + CPUEntry* cpu = core->CPUHeap()->PeekRoot(); + ASSERT(cpu != NULL); + + if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { + cpu->UpdatePriority(threadPriority); + rescheduleNeeded = true; + } else + rescheduleNeeded = false; + + return cpu; +} + + +inline bigtime_t +ThreadData::_GetBaseQuantum() const +{ + SCHEDULER_ENTER_FUNCTION(); + + return sQuantumLengths[GetEffectivePriority()]; +} + + ThreadData::ThreadData(Thread* thread) : fThread(thread) @@ -190,6 +242,21 @@ ThreadData::ComputeQuantumLengths() } +inline int32 +ThreadData::_GetPenalty() const +{ + SCHEDULER_ENTER_FUNCTION(); + + int32 penalty = fPriorityPenalty; + + const int kMinimalPriority = _GetMinimalPriority(); + if (kMinimalPriority > 0) + penalty += fAdditionalPenalty % kMinimalPriority; + + return penalty; +} + + void ThreadData::_ComputeEffectivePriority() const { @@ -209,58 +276,6 @@ ThreadData::_ComputeEffectivePriority() const } -inline CoreEntry* -ThreadData::_ChooseCore() const -{ - SCHEDULER_ENTER_FUNCTION(); - - ASSERT(!gSingleCore); - return gCurrentMode->choose_core(this); -} - - -inline CPUEntry* -ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const -{ - SCHEDULER_ENTER_FUNCTION(); - - int32 threadPriority = GetEffectivePriority(); - - if (fThread->previous_cpu != NULL) { - CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; - if (previousCPU->Core() == core && !fThread->previous_cpu->disabled) { - CoreCPUHeapLocker _(core); - if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { - previousCPU->UpdatePriority(threadPriority); - rescheduleNeeded = true; - return previousCPU; - } - } - } - - CoreCPUHeapLocker _(core); - CPUEntry* cpu = core->CPUHeap()->PeekRoot(); - ASSERT(cpu != NULL); - - if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { - cpu->UpdatePriority(threadPriority); - rescheduleNeeded = true; - } else - rescheduleNeeded = false; - - return cpu; -} - - -inline bigtime_t -ThreadData::_GetBaseQuantum() const -{ - SCHEDULER_ENTER_FUNCTION(); - - return sQuantumLengths[GetEffectivePriority()]; -} - - /* static */ bigtime_t ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, int32 maxPriority, int32 minPriority, int32 priority) diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index debcf2d98b..de932dd1dd 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -11,6 +11,7 @@ #include "scheduler_common.h" #include "scheduler_cpu.h" +#include "scheduler_locking.h" #include "scheduler_profiler.h" @@ -19,6 +20,17 @@ namespace Scheduler { struct ThreadData : public DoublyLinkedListLinkImpl, RunQueueLinkImpl { +private: + inline void _InitBase(); + + inline int32 _GetMinimalPriority() const; + + inline CoreEntry* _ChooseCore() const; + inline CPUEntry* _ChooseCPU(CoreEntry* core, + bool& rescheduleNeeded) const; + + inline bigtime_t _GetBaseQuantum() const; + public: ThreadData(Thread* thread); @@ -72,17 +84,12 @@ public: inline void UnassignCore() { fCore = NULL; } static void ComputeQuantumLengths(); + private: inline int32 _GetPenalty() const; - inline int32 _GetMinimalPriority() const; void _ComputeEffectivePriority() const; - inline CoreEntry* _ChooseCore() const; - inline CPUEntry* _ChooseCPU(CoreEntry* core, - bool& rescheduleNeeded) const; - - inline bigtime_t _GetBaseQuantum() const; static bigtime_t _ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, int32 maxPriority, int32 minPriority, int32 priority); @@ -121,6 +128,21 @@ public: }; +inline int32 +ThreadData::_GetMinimalPriority() const +{ + SCHEDULER_ENTER_FUNCTION(); + + const int32 kDivisor = 5; + + const int32 kMaximalPriority = 25; + const int32 kMinimalPriority = B_LOWEST_ACTIVE_PRIORITY; + + int32 priority = fThread->priority / kDivisor; + return std::max(std::min(priority, kMaximalPriority), kMinimalPriority); +} + + inline bool ThreadData::HasCacheExpired() const { @@ -259,6 +281,7 @@ ThreadData::Enqueue() SCHEDULER_ENTER_FUNCTION(); fThread->state = B_THREAD_READY; + ComputeLoad(); fWentSleepCount = 0; @@ -350,36 +373,6 @@ ThreadData::StartQuantum() } -inline int32 -ThreadData::_GetPenalty() const -{ - SCHEDULER_ENTER_FUNCTION(); - - int32 penalty = fPriorityPenalty; - - const int kMinimalPriority = _GetMinimalPriority(); - if (kMinimalPriority > 0) - penalty += fAdditionalPenalty % kMinimalPriority; - - return penalty; -} - - -inline int32 -ThreadData::_GetMinimalPriority() const -{ - SCHEDULER_ENTER_FUNCTION(); - - const int32 kDivisor = 5; - - const int32 kMaximalPriority = 25; - const int32 kMinimalPriority = B_LOWEST_ACTIVE_PRIORITY; - - int32 priority = fThread->priority / kDivisor; - return std::max(std::min(priority, kMaximalPriority), kMinimalPriority); -} - - } // namespace Scheduler From 8235bbc9965b083b294b366ea5438d2ff274dbf7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 6 Jan 2014 01:18:37 +0100 Subject: [PATCH 254/273] scheduler: Improve thread creation performance --- src/system/kernel/scheduler/scheduler.cpp | 2 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 28 +++++--- src/system/kernel/scheduler/scheduler_cpu.h | 10 --- .../kernel/scheduler/scheduler_thread.cpp | 68 ++++++++----------- .../kernel/scheduler/scheduler_thread.h | 24 ++++--- 5 files changed, 64 insertions(+), 68 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 2916b5b9aa..646c756ad6 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -383,7 +383,7 @@ reschedule(int32 nextState) int32 thisCPU = smp_get_current_cpu(); - CPUEntry* cpu = &gCPUEntries[thisCPU]; + CPUEntry* cpu = CPUEntry::GetCPU(thisCPU); CoreEntry* core = CoreEntry::GetCore(thisCPU); TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 7ec4ab5eec..771f1d799d 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -123,6 +123,14 @@ CPUEntry::Remove(ThreadData* thread) } +inline ThreadData* +CoreEntry::PeekThread() const +{ + SCHEDULER_ENTER_FUNCTION(); + return fRunQueue.PeekMaximum(); +} + + inline ThreadData* CPUEntry::PeekThread() const { @@ -152,6 +160,9 @@ CPUEntry::UpdatePriority(int32 priority) return; fCore->CPUHeap()->ModifyKey(this, priority); + if (gSingleCore) + return; + if (oldPriority == B_IDLE_PRIORITY) fCore->CPUWakesUp(this); else if (priority == B_IDLE_PRIORITY) @@ -164,6 +175,7 @@ CPUEntry::ComputeLoad() { SCHEDULER_ENTER_FUNCTION(); + ASSERT(gTrackLoad); ASSERT(fCPUNumber == smp_get_current_cpu()); int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad); @@ -244,10 +256,12 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) oldThreadData->UpdateActivity(active); } - oldThreadData->ComputeLoad(); - nextThreadData->ComputeLoad(); - if (gTrackLoad && !cpuEntry->disabled) - ComputeLoad(); + if (gTrackLoad) { + oldThreadData->ComputeLoad(); + nextThreadData->ComputeLoad(); + if (!cpuEntry->disabled) + ComputeLoad(); + } Thread* nextThread = nextThreadData->GetThread(); if (!thread_is_idle_thread(nextThread)) { @@ -256,19 +270,17 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) nextThreadData->SetLastInterruptTime(cpuEntry->interrupt_time); + if (gCPUFrequencyManagement) _RequestPerformanceLevel(nextThreadData); } } -inline void +void CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) { SCHEDULER_ENTER_FUNCTION(); - if (!gCPUFrequencyManagement) - return; - if (gCPU[fCPUNumber].disabled) { decrease_cpu_performance(kCPUPerformanceScaleMax); return; diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index b62ee5f3a2..1def98d4e6 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -112,7 +112,6 @@ public: void Dump(); }; - class CoreEntry : public MinMaxHeapLinkImpl, public DoublyLinkedListLinkImpl { public: @@ -190,7 +189,6 @@ private: friend class DebugDumper; } CACHE_LINE_ALIGN; - class CoreLoadHeap : public MinMaxHeap { public: CoreLoadHeap() { } @@ -356,14 +354,6 @@ CoreEntry::IncreaseActiveTime(bigtime_t activeTime) } -inline ThreadData* -CoreEntry::PeekThread() const -{ - SCHEDULER_ENTER_FUNCTION(); - return fRunQueue.PeekMaximum(); -} - - inline bigtime_t CoreEntry::GetActiveTime() const { diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index aef21a59f1..6f9c818dad 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -12,6 +12,28 @@ using namespace Scheduler; static bigtime_t sQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; +void +ThreadData::_InitBase() +{ + fPriorityPenalty = 0; + fAdditionalPenalty = 0; + fEffectivePriority = fThread->priority; + + fTimeLeft = 0; + fStolenTime = 0; + + fMeasureActiveTime = 0; + fMeasureTime = 0; + fLoad = 0; + + fWentSleep = 0; + fWentSleepActive = 0; + fWentSleepCount = -1; + + fEnqueued = false; +} + + inline CoreEntry* ThreadData::_ChooseCore() const { @@ -30,7 +52,8 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const int32 threadPriority = GetEffectivePriority(); if (fThread->previous_cpu != NULL) { - CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; + CPUEntry* previousCPU + = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num); if (previousCPU->Core() == core && !fThread->previous_cpu->disabled) { CoreCPUHeapLocker _(core); if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { @@ -74,48 +97,18 @@ ThreadData::ThreadData(Thread* thread) void ThreadData::Init() { - fAdditionalPenalty = 0; - fEffectivePriority = -1; + _InitBase(); - fTimeLeft = 0; - fStolenTime = 0; - - fMeasureActiveTime = 0; - fMeasureTime = 0; - fLoad = 0; - - fWentSleep = 0; - fWentSleepActive = 0; - fWentSleepCount = -1; - - fEnqueued = false; - - Thread* currentThread = thread_get_current_thread(); - ASSERT(currentThread != NULL); - if (!thread_is_idle_thread(currentThread)) { - ThreadData* currentThreadData = currentThread->scheduler_data; - int32 penalty = currentThreadData->fPriorityPenalty; - - int32 minimalPriority = _GetMinimalPriority(); - if (fThread->priority - penalty >= minimalPriority) - fPriorityPenalty = penalty; - else - fPriorityPenalty = fThread->priority - minimalPriority; - - fCore = currentThreadData->fCore; - } else { - fPriorityPenalty = 0; - fAdditionalPenalty = 0; - - fCore = NULL; - } + ThreadData* currentThreadData = thread_get_current_thread()->scheduler_data; + fCore = currentThreadData->fCore; } void ThreadData::Init(CoreEntry* core) { - Init(); + _InitBase(); + fCore = core; } @@ -172,8 +165,7 @@ ThreadData::ComputeLoad() { SCHEDULER_ENTER_FUNCTION(); - if (!gTrackLoad) - return; + ASSERT(gTrackLoad); if (fLastInterruptTime > 0) { bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index de932dd1dd..8d9cbdda2d 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -165,9 +165,6 @@ inline int32 ThreadData::GetEffectivePriority() const { SCHEDULER_ENTER_FUNCTION(); - - if (fEffectivePriority == -1) - _ComputeEffectivePriority(); return fEffectivePriority; } @@ -184,7 +181,6 @@ ThreadData::IncreasePenalty() TRACE("increasing thread %ld penalty\n", fThread->id); - fEffectivePriority = -1; int32 oldPenalty = fPriorityPenalty++; ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); @@ -194,6 +190,8 @@ ThreadData::IncreasePenalty() fPriorityPenalty = oldPenalty; fAdditionalPenalty++; } + + _ComputeEffectivePriority(); } @@ -202,13 +200,15 @@ ThreadData::CancelPenalty() { SCHEDULER_ENTER_FUNCTION(); - if (fPriorityPenalty != 0) { - TRACE("cancelling thread %ld penalty\n", fThread->id); - fEffectivePriority = -1; - } + int32 oldPenalty = fPriorityPenalty; fAdditionalPenalty = 0; fPriorityPenalty = 0; + + if (oldPenalty != 0) { + TRACE("cancelling thread %ld penalty\n", fThread->id); + _ComputeEffectivePriority(); + } } @@ -217,7 +217,7 @@ ThreadData::ShouldCancelPenalty() const { SCHEDULER_ENTER_FUNCTION(); - if (fCore == NULL || fWentSleep == 0) + if (fCore == NULL) return false; return fCore->StarvationCounter() != fWentSleepCount @@ -251,7 +251,8 @@ ThreadData::PutBack() { SCHEDULER_ENTER_FUNCTION(); - ComputeLoad(); + if (gTrackLoad) + ComputeLoad(); fWentSleepCount = -1; int32 priority = GetEffectivePriority(); @@ -282,7 +283,8 @@ ThreadData::Enqueue() fThread->state = B_THREAD_READY; - ComputeLoad(); + if (gTrackLoad) + ComputeLoad(); fWentSleepCount = 0; int32 priority = GetEffectivePriority(); From 8cf8e537740789b1b103f0aa0736dbfcf55359c2 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 6 Jan 2014 06:49:34 +0100 Subject: [PATCH 255/273] kernel/x86: Inline atomic functions and memory barriers --- headers/private/kernel/arch/atomic.h | 22 +++ headers/private/kernel/arch/cpu.h | 4 - headers/private/kernel/arch/x86/32/atomic.h | 97 +++++++++++++ headers/private/kernel/arch/x86/64/atomic.h | 153 ++++++++++++++++++++ headers/private/kernel/lock.h | 3 + headers/private/kernel/smp.h | 7 +- headers/private/kernel/thread.h | 5 +- src/system/kernel/arch/x86/arch_cpu.cpp | 35 ----- src/system/kernel/arch/x86/arch_smp.cpp | 7 +- src/system/kernel/smp.cpp | 23 ++- 10 files changed, 301 insertions(+), 55 deletions(-) create mode 100644 headers/private/kernel/arch/atomic.h create mode 100644 headers/private/kernel/arch/x86/32/atomic.h create mode 100644 headers/private/kernel/arch/x86/64/atomic.h diff --git a/headers/private/kernel/arch/atomic.h b/headers/private/kernel/arch/atomic.h new file mode 100644 index 0000000000..c958b3c6d6 --- /dev/null +++ b/headers/private/kernel/arch/atomic.h @@ -0,0 +1,22 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_ARCH_ATOMIC_H +#define _KERNEL_ARCH_ATOMIC_H + + +#include + +#include + + +#ifdef __x86_64__ +# include +#elif __INTEL__ +# include +#endif + + +#endif // _KERNEL_ARCH_ATOMIC_H + diff --git a/headers/private/kernel/arch/cpu.h b/headers/private/kernel/arch/cpu.h index d5ea2de81e..188df59081 100644 --- a/headers/private/kernel/arch/cpu.h +++ b/headers/private/kernel/arch/cpu.h @@ -43,10 +43,6 @@ status_t arch_cpu_user_memset(void *s, char c, size_t count, void arch_cpu_sync_icache(void *address, size_t length); -void arch_cpu_memory_read_barrier(void); -void arch_cpu_memory_write_barrier(void); -void arch_cpu_memory_read_write_barrier(void); - #ifdef __cplusplus } diff --git a/headers/private/kernel/arch/x86/32/atomic.h b/headers/private/kernel/arch/x86/32/atomic.h new file mode 100644 index 0000000000..adec9c05c3 --- /dev/null +++ b/headers/private/kernel/arch/x86/32/atomic.h @@ -0,0 +1,97 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_ARCH_X86_32_ATOMIC_H +#define _KERNEL_ARCH_X86_32_ATOMIC_H + + +static inline void +memory_read_barrier_inline(void) +{ + asm volatile("lock; addl $0, (%%esp)" : : : "memory"); +} + + +static inline void +memory_write_barrier_inline(void) +{ + asm volatile("lock; addl $0, (%%esp)" : : : "memory"); +} + + +static inline void +memory_full_barrier_inline(void) +{ + asm volatile("lock; addl $0, (%%esp)" : : : "memory"); +} + + +#define memory_read_barrier memory_read_barrier_inline +#define memory_write_barrier memory_write_barrier_inline +#define memory_full_barrier memory_full_barrier_inline + + +static inline void +atomic_set_inline(int32* value, int32 newValue) +{ + memory_write_barrier(); + *(volatile int32*)value = newValue; +} + + +static inline int32 +atomic_get_and_set_inline(int32* value, int32 newValue) +{ + asm volatile("xchgl %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst) +{ + asm volatile("lock; cmpxchgl %2, (%3)" + : "=a" (newValue) + : "0" (testAgainst), "r" (newValue), "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_add_inline(int32* value, int32 newValue) +{ + asm volatile("lock; xaddl %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_get_inline(int32* value) +{ + int32 newValue = *(volatile int32*)value; + memory_read_barrier(); + return newValue; +} + + +#define atomic_set atomic_set_inline +#define atomic_get_and_set atomic_get_and_set_inline +#ifndef atomic_test_and_set +# define atomic_test_and_set atomic_test_and_set_inline +#endif +#ifndef atomic_add +# define atomic_add atomic_add_inline +#endif +#define atomic_get atomic_get_inline + + +#endif // _KERNEL_ARCH_X86_32_ATOMIC_H + diff --git a/headers/private/kernel/arch/x86/64/atomic.h b/headers/private/kernel/arch/x86/64/atomic.h new file mode 100644 index 0000000000..fd567319f6 --- /dev/null +++ b/headers/private/kernel/arch/x86/64/atomic.h @@ -0,0 +1,153 @@ +/* + * Copyright 2014, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef _KERNEL_ARCH_X86_64_ATOMIC_H +#define _KERNEL_ARCH_X86_64_ATOMIC_H + + +static inline void +memory_read_barrier_inline(void) +{ + asm volatile("lfence" : : : "memory"); +} + + +static inline void +memory_write_barrier_inline(void) +{ + asm volatile("sfence" : : : "memory"); +} + + +static inline void +memory_full_barrier_inline(void) +{ + asm volatile("mfence" : : : "memory"); +} + + +#define memory_read_barrier memory_read_barrier_inline +#define memory_write_barrier memory_write_barrier_inline +#define memory_full_barrier memory_full_barrier_inline + + +static inline void +atomic_set_inline(int32* value, int32 newValue) +{ + memory_write_barrier(); + *(volatile int32*)value = newValue; +} + + +static inline int32 +atomic_get_and_set_inline(int32* value, int32 newValue) +{ + asm volatile("xchg %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_test_and_set_inline(int32* value, int32 newValue, int32 testAgainst) +{ + asm volatile("lock; cmpxchgl %2, (%3)" + : "=a" (newValue) + : "0" (testAgainst), "r" (newValue), "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_add_inline(int32* value, int32 newValue) +{ + asm volatile("lock; xaddl %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int32 +atomic_get_inline(int32* value) +{ + int32 newValue = *(volatile int32*)value; + memory_read_barrier(); + return newValue; +} + + +static inline void +atomic_set64_inline(int64* value, int64 newValue) +{ + memory_write_barrier(); + *(volatile int64*)value = newValue; +} + + +static inline int64 +atomic_get_and_set64_inline(int64* value, int64 newValue) +{ + asm volatile("xchgq %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int64 +atomic_test_and_set64_inline(int64* value, int64 newValue, int64 testAgainst) +{ + asm volatile("lock; cmpxchgq %2, (%3)" + : "=a" (newValue) + : "0" (testAgainst), "r" (newValue), "r" (value) + : "memory"); + return newValue; +} + + +static inline int64 +atomic_add64_inline(int64* value, int64 newValue) +{ + asm volatile("lock; xaddq %0, (%1)" + : "+r" (newValue) + : "r" (value) + : "memory"); + return newValue; +} + + +static inline int64 +atomic_get64_inline(int64* value) +{ + int64 newValue = *(volatile int64*)value; + memory_read_barrier(); + return newValue; +} + + +#define atomic_set atomic_set_inline +#define atomic_get_and_set atomic_get_and_set_inline +#ifndef atomic_test_and_set +# define atomic_test_and_set atomic_test_and_set_inline +#endif +#ifndef atomic_add +# define atomic_add atomic_add_inline +#endif +#define atomic_get atomic_get_inline + +#define atomic_set64 atomic_set64_inline +#define atomic_get_and_set64 atomic_get_and_set64_inline +#define atomic_test_and_set64 atomic_test_and_set64_inline +#define atomic_add64 atomic_add64_inline +#define atomic_get64 atomic_get64_inline + + +#endif // _KERNEL_ARCH_X86_64_ATOMIC_H + diff --git a/headers/private/kernel/lock.h b/headers/private/kernel/lock.h index 64667a72f7..d3534aa188 100644 --- a/headers/private/kernel/lock.h +++ b/headers/private/kernel/lock.h @@ -9,7 +9,10 @@ #ifndef _KERNEL_LOCK_H #define _KERNEL_LOCK_H + #include + +#include #include diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 3378f23d26..685e3d5a03 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -9,6 +9,7 @@ #define KERNEL_SMP_H +#include #include #include @@ -185,7 +186,7 @@ release_spinlock_inline(spinlock* lock) static inline bool try_acquire_write_spinlock_inline(rw_spinlock* lock) { - return atomic_test_and_set(&lock->lock, 1 << 31, 0) == 0; + return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; } @@ -209,9 +210,9 @@ static inline bool try_acquire_read_spinlock_inline(rw_spinlock* lock) { uint32 previous = atomic_add(&lock->lock, 1); - if ((previous & (1 << 31)) == 0) + if ((previous & (1u << 31)) == 0) return true; - atomic_test_and_set(&lock->lock, 1 << 31, previous); + atomic_test_and_set(&lock->lock, 1u << 31, previous); return false; } diff --git a/headers/private/kernel/thread.h b/headers/private/kernel/thread.h index f41987c8c4..f502c94251 100644 --- a/headers/private/kernel/thread.h +++ b/headers/private/kernel/thread.h @@ -11,12 +11,13 @@ #include -#include -#include +#include +#include // For the thread blocking inline functions only. #include #include +#include struct arch_fork_arg; diff --git a/src/system/kernel/arch/x86/arch_cpu.cpp b/src/system/kernel/arch/x86/arch_cpu.cpp index 3de6d53835..a21afcde52 100644 --- a/src/system/kernel/arch/x86/arch_cpu.cpp +++ b/src/system/kernel/arch/x86/arch_cpu.cpp @@ -1288,38 +1288,3 @@ arch_cpu_sync_icache(void* address, size_t length) // instruction cache is always consistent on x86 } - -void -arch_cpu_memory_read_barrier(void) -{ -#ifdef __x86_64__ - asm volatile("lfence" : : : "memory"); -#else - asm volatile ("lock;" : : : "memory"); - asm volatile ("addl $0, 0(%%esp);" : : : "memory"); -#endif -} - - -void -arch_cpu_memory_write_barrier(void) -{ -#ifdef __x86_64__ - asm volatile("sfence" : : : "memory"); -#else - asm volatile ("lock;" : : : "memory"); - asm volatile ("addl $0, 0(%%esp);" : : : "memory"); -#endif -} - - -void -arch_cpu_memory_read_write_barrier(void) -{ -#ifdef __x86_64__ - asm volatile("mfence" : : : "memory"); -#else - asm volatile ("lock;" : : : "memory"); - asm volatile ("addl $0, 0(%%esp);" : : : "memory"); -#endif -} diff --git a/src/system/kernel/arch/x86/arch_smp.cpp b/src/system/kernel/arch/x86/arch_smp.cpp index e9d4b4c545..fd5535157f 100644 --- a/src/system/kernel/arch/x86/arch_smp.cpp +++ b/src/system/kernel/arch/x86/arch_smp.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -139,7 +140,7 @@ arch_smp_send_multicast_ici(CPUSet& cpuSet) panic("arch_smp_send_multicast_ici: called with interrupts enabled"); #endif - arch_cpu_memory_write_barrier(); + memory_write_barrier(); int32 i = 0; int32 cpuCount = smp_get_num_cpus(); @@ -189,7 +190,7 @@ arch_smp_send_broadcast_ici(void) panic("arch_smp_send_broadcast_ici: called with interrupts enabled"); #endif - arch_cpu_memory_write_barrier(); + memory_write_barrier(); uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED | APIC_INTR_COMMAND_1_ASSERT @@ -210,7 +211,7 @@ arch_smp_send_ici(int32 target_cpu) panic("arch_smp_send_ici: called with interrupts enabled"); #endif - arch_cpu_memory_write_barrier(); + memory_write_barrier(); uint32 destination = sCPUAPICIds[target_cpu]; uint32 mode = ICI_VECTOR | APIC_DELIVERY_MODE_FIXED diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index 3631234771..a82c7a273e 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -616,10 +617,10 @@ try_acquire_read_spinlock(rw_spinlock* lock) #endif uint32 previous = atomic_add(&lock->lock, 1); - if ((previous & (1 << 31)) == 0) + if ((previous & (1u << 31)) == 0) return true; - atomic_test_and_set(&lock->lock, 1 << 31, previous); + atomic_test_and_set(&lock->lock, 1u << 31, previous); return false; } @@ -659,7 +660,7 @@ release_read_spinlock(rw_spinlock* lock) { #if DEBUG_SPINLOCKS uint32 previous = atomic_add(&lock->lock, -1); - if ((previous & 1 << 31) != 0) { + if ((previous & 1u << 31) != 0) { panic("release_read_spinlock: lock %p was already released (value:" " %#" B_PRIx32 ")\n", lock, previous); } @@ -701,7 +702,8 @@ acquire_read_seqlock(seqlock* lock) { bool release_read_seqlock(seqlock* lock, uint32 count) { - arch_cpu_memory_read_barrier(); + memory_read_barrier(); + uint32 current = atomic_get((int32*)&lock->count); if (count % 2 == 1 || current != count) { @@ -1517,15 +1519,20 @@ call_all_cpus_sync(void (*func)(void*, int), void* cookie) } +#undef memory_read_barrier +#undef memory_write_barrier + + void -memory_read_barrier(void) +memory_read_barrier() { - arch_cpu_memory_read_barrier(); + memory_read_barrier_inline(); } void -memory_write_barrier(void) +memory_write_barrier() { - arch_cpu_memory_write_barrier(); + memory_write_barrier_inline(); } + From a5f45afa6c2f39d00951d01a0a4a2865b4b80059 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 6 Jan 2014 07:35:36 +0100 Subject: [PATCH 256/273] scheduler: Remove unnecessary check against disabled CPU --- src/system/kernel/scheduler/scheduler.cpp | 8 ++++---- src/system/kernel/scheduler/scheduler_cpu.cpp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 646c756ad6..ca67208868 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -445,14 +445,14 @@ reschedule(int32 nextState) nextThreadData = cpu->ChooseNextThread(enqueueOldThread ? oldThreadData : NULL, putOldThreadAtBack); + + // update CPU heap + CoreCPUHeapLocker cpuLocker(core); + cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); } Thread* nextThread = nextThreadData->GetThread(); ASSERT(!gCPU[thisCPU].disabled || thread_is_idle_thread(nextThread)); - // update CPU heap - CoreCPUHeapLocker cpuLocker(core); - cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); - cpuLocker.Unlock(); if (nextThread != oldThread) { if (enqueueOldThread) { diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 771f1d799d..9f9eb48cef 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -152,8 +152,7 @@ CPUEntry::UpdatePriority(int32 priority) { SCHEDULER_ENTER_FUNCTION(); - if (gCPU[fCPUNumber].disabled) - return; + ASSERT(!gCPU[fCPUNumber].disabled); int32 oldPriority = CPUPriorityHeap::GetKey(this); if (oldPriority == priority) @@ -176,6 +175,7 @@ CPUEntry::ComputeLoad() SCHEDULER_ENTER_FUNCTION(); ASSERT(gTrackLoad); + ASSERT(!gCPU[fCPUNumber].disabled); ASSERT(fCPUNumber == smp_get_current_cpu()); int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad); From bae6d7196a35aa0822c5f511c80b2ca6e70c009e Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 6 Jan 2014 09:05:34 +0100 Subject: [PATCH 257/273] scheduler: Fix SchedulerModeLocker --- src/system/kernel/scheduler/scheduler_locking.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler_locking.h b/src/system/kernel/scheduler/scheduler_locking.h index 044015c236..8feffe60ed 100644 --- a/src/system/kernel/scheduler/scheduler_locking.h +++ b/src/system/kernel/scheduler/scheduler_locking.h @@ -82,10 +82,13 @@ class SchedulerModeLocker : public: SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) : - AutoLocker(NULL, alreadyLocked, + AutoLocker(&fDummy, alreadyLocked, lockIfNotLocked) { } + +private: + int fDummy; }; class InterruptsSchedulerModeLocking { From 4ca31ac964da8520e1804b7cfb1f4d4479a80497 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Mon, 6 Jan 2014 21:02:11 +0100 Subject: [PATCH 258/273] kernel: Fix ABA problem in try_acquire_read_spinlock() --- headers/private/kernel/smp.h | 5 +---- src/system/kernel/smp.cpp | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/headers/private/kernel/smp.h b/headers/private/kernel/smp.h index 685e3d5a03..43126eb697 100644 --- a/headers/private/kernel/smp.h +++ b/headers/private/kernel/smp.h @@ -210,10 +210,7 @@ static inline bool try_acquire_read_spinlock_inline(rw_spinlock* lock) { uint32 previous = atomic_add(&lock->lock, 1); - if ((previous & (1u << 31)) == 0) - return true; - atomic_test_and_set(&lock->lock, 1u << 31, previous); - return false; + return (previous & (1u << 31)) == 0; } diff --git a/src/system/kernel/smp.cpp b/src/system/kernel/smp.cpp index a82c7a273e..325fa7206d 100644 --- a/src/system/kernel/smp.cpp +++ b/src/system/kernel/smp.cpp @@ -617,11 +617,7 @@ try_acquire_read_spinlock(rw_spinlock* lock) #endif uint32 previous = atomic_add(&lock->lock, 1); - if ((previous & (1u << 31)) == 0) - return true; - - atomic_test_and_set(&lock->lock, 1u << 31, previous); - return false; + return (previous & (1u << 31)) == 0; } From c37c2aa45fd02af0fa4941751bdabf5cbcc664d6 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 7 Jan 2014 01:09:56 +0100 Subject: [PATCH 259/273] scheduler: Improve should_rebalance --- src/system/kernel/scheduler/low_latency.cpp | 16 ++++++++++------ src/system/kernel/scheduler/power_saving.cpp | 16 ++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 42baee3325..09c6f719e8 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -84,14 +84,18 @@ should_rebalance(const ThreadData* threadData) if (threadData->GetLoad() >= coreLoad / 2) return false; + int32 threadLoad = threadData->GetLoad(); + int32 coreNewLoad = coreLoad - threadLoad; + // If there is high load on this core but this thread does not contribute // significantly consider giving it to someone less busy. if (coreLoad > kHighLoad) { - ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap.PeekMinimum(); - if (other != NULL && coreLoad - other->GetLoad() >= kLoadDifference) - return true; + if (other != NULL) { + int32 otherNewLoad = other->GetLoad() + threadLoad; + if (coreNewLoad - otherNewLoad >= kLoadDifference) + return true; + } } // No cpu bound threads - the situation is quite good. Make sure it @@ -101,8 +105,8 @@ should_rebalance(const ThreadData* threadData) CoreEntry* other = gCoreLoadHeap.PeekMinimum(); if (other == NULL) other = gCoreHighLoadHeap.PeekMinimum(); - ASSERT(other != NULL); - return coreLoad - other->GetLoad() >= kLoadDifference * 2; + int32 otherNewLoad = other->GetLoad() + threadLoad; + return coreNewLoad - otherNewLoad >= kLoadDifference * 2; } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 3b36add0c9..b41bbe7007 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -52,10 +52,7 @@ choose_small_task_core() { SCHEDULER_ENTER_FUNCTION(); - ReadSpinLocker locker(gCoreHeapsLock); CoreEntry* core = gCoreLoadHeap.PeekMaximum(); - locker.Unlock(); - if (core == NULL) return sSmallTaskCore; @@ -126,25 +123,28 @@ should_rebalance(const ThreadData* threadData) CoreEntry* core = threadData->Core(); int32 coreLoad = core->GetLoad(); + int32 threadLoad = threadData->GetLoad(); if (coreLoad > kHighLoad) { - ReadSpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { sSmallTaskCore = NULL; choose_small_task_core(); - if (threadData->GetLoad() > coreLoad / 3) + if (threadLoad > coreLoad / 3) return false; return coreLoad > kVeryHighLoad; } - if (threadData->GetLoad() >= coreLoad / 2) + if (threadLoad >= coreLoad / 2) return false; CoreEntry* other = gCoreLoadHeap.PeekMaximum(); if (other == NULL) other = gCoreHighLoadHeap.PeekMinimum(); ASSERT(other != NULL); - return coreLoad - other->GetLoad() >= kLoadDifference / 2; + + int32 coreNewLoad = coreLoad - threadLoad; + int32 otherNewLoad = other->GetLoad() + threadLoad; + return coreNewLoad - otherNewLoad >= kLoadDifference / 2; } if (coreLoad >= kMediumLoad) @@ -154,7 +154,7 @@ should_rebalance(const ThreadData* threadData) if (smallTaskCore == NULL) return false; return smallTaskCore != core - && smallTaskCore->GetLoad() +threadData->GetLoad() < kHighLoad; + && smallTaskCore->GetLoad() + threadLoad < kHighLoad; } From c2a02dee65184026ea953726a9ab1bac1c0a4617 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Tue, 7 Jan 2014 02:09:29 +0100 Subject: [PATCH 260/273] kernel: Relax cpu_ent::interrupt_time locking The value isn't accessed by the other CPUs and all writes and reads are done with interrupts disabled. --- src/system/kernel/int.cpp | 10 ++++++---- src/system/kernel/scheduler/scheduler.cpp | 3 +-- src/system/kernel/scheduler/scheduler_thread.cpp | 7 ------- src/system/kernel/scheduler/scheduler_thread.h | 12 ++++++------ 4 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index fea008b845..c09d8bc6c3 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -359,14 +359,16 @@ int_io_interrupt_handler(int vector, bool levelTriggered) if (!sVectors[vector].no_lock_vector) release_spinlock(&sVectors[vector].vector_lock); - SpinLocker locker(sVectors[vector].load_lock); + SpinLocker vectorLocker(sVectors[vector].load_lock); bigtime_t deltaTime = system_time() - start; sVectors[vector].last_measure_active += deltaTime; - locker.Unlock(); + vectorLocker.Unlock(); - atomic_add64(&get_cpu_struct()->interrupt_time, deltaTime); + cpu_ent* cpu = get_cpu_struct(); + cpu->interrupt_time += deltaTime; if (sVectors[vector].type == INTERRUPT_TYPE_IRQ) - atomic_add64(&get_cpu_struct()->irq_time, deltaTime); + cpu->irq_time += deltaTime; + update_int_load(vector); if (levelTriggered) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ca67208868..ca04585c92 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -393,8 +393,7 @@ reschedule(int32 nextState) ThreadData* oldThreadData = oldThread->scheduler_data; // return time spent in interrupts - oldThreadData->IncreaseStolenTime( - gCPU[thisCPU].interrupt_time - oldThreadData->LastInterruptTime()); + oldThreadData->SetStolenInterruptTime(gCPU[thisCPU].interrupt_time); bool enqueueOldThread = false; bool putOldThreadAtBack = false; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 6f9c818dad..af4b40aa1e 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -166,13 +166,6 @@ ThreadData::ComputeLoad() SCHEDULER_ENTER_FUNCTION(); ASSERT(gTrackLoad); - - if (fLastInterruptTime > 0) { - bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; - interruptTime -= fLastInterruptTime; - fMeasureActiveTime -= interruptTime; - } - compute_load(fMeasureTime, fMeasureActiveTime, fLoad); } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 8d9cbdda2d..5c8d052120 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -51,12 +51,9 @@ public: bool ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU); - inline bigtime_t LastInterruptTime() const - { return fLastInterruptTime; } inline void SetLastInterruptTime(bigtime_t interruptTime) { fLastInterruptTime = interruptTime; } - - inline void IncreaseStolenTime(bigtime_t stolenTime); + inline void SetStolenInterruptTime(bigtime_t interruptTime); inline void GoesAway(); inline bigtime_t WentSleep() const { return fWentSleep; } @@ -226,10 +223,13 @@ ThreadData::ShouldCancelPenalty() const inline void -ThreadData::IncreaseStolenTime(bigtime_t stolenTime) +ThreadData::SetStolenInterruptTime(bigtime_t interruptTime) { SCHEDULER_ENTER_FUNCTION(); - fStolenTime += stolenTime; + + interruptTime -= fLastInterruptTime; + fStolenTime += interruptTime; + fMeasureActiveTime -= interruptTime; } From 9c465cc83bbd40732475db43bd870221b99bdbb7 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 8 Jan 2014 04:36:28 +0100 Subject: [PATCH 261/273] scheduler: Improve recognition of CPU bound threads --- src/system/kernel/scheduler/low_latency.cpp | 8 ++-- src/system/kernel/scheduler/power_saving.cpp | 2 - src/system/kernel/scheduler/scheduler.cpp | 3 -- src/system/kernel/scheduler/scheduler_cpu.cpp | 31 ++++++------- src/system/kernel/scheduler/scheduler_cpu.h | 22 +++++++-- src/system/kernel/scheduler/scheduler_modes.h | 2 - .../kernel/scheduler/scheduler_thread.cpp | 26 ++++++++++- .../kernel/scheduler/scheduler_thread.h | 45 ++++++++++++++----- 8 files changed, 95 insertions(+), 44 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 09c6f719e8..51381c3ac8 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -160,13 +160,11 @@ rebalance_irqs(bool idle) scheduler_mode_operations gSchedulerLowLatencyMode = { "low latency", - true, - 2000, - 700, - { 2, 30 }, + 100, + { 2, 25 }, - 60000, + 50000, switch_to_mode, set_cpu_enabled, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index b41bbe7007..66efc8ce9b 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -236,8 +236,6 @@ rebalance_irqs(bool idle) scheduler_mode_operations gSchedulerPowerSavingMode = { "power saving", - false, - 3000, 1000, { 3, 60 }, diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ca04585c92..0ee4cd0bd8 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -404,8 +404,6 @@ reschedule(int32 nextState) if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, oldThread->has_yielded)) { - oldThreadData->IncreasePenalty(); - TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, oldThreadData->GetEffectivePriority()); putOldThreadAtBack = true; @@ -419,7 +417,6 @@ reschedule(int32 nextState) case THREAD_STATE_FREE_ON_RESCHED: break; default: - oldThreadData->IncreasePenalty(); oldThreadData->GoesAway(); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, nextState); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 9f9eb48cef..4bdb566840 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -21,7 +21,6 @@ public: static void DumpCPURunQueue(CPUEntry* cpu); static void DumpCoreRunQueue(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); - }; @@ -43,7 +42,8 @@ ThreadRunQueue::Dump() const kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", thread, thread->id, thread->priority, - threadData->GetEffectivePriority(), thread->name); + thread->priority - threadData->GetEffectivePriority(), + thread->name); } } } @@ -159,9 +159,6 @@ CPUEntry::UpdatePriority(int32 priority) return; fCore->CPUHeap()->ModifyKey(this, priority); - if (gSingleCore) - return; - if (oldPriority == B_IDLE_PRIORITY) fCore->CPUWakesUp(this); else if (priority == B_IDLE_PRIORITY) @@ -345,6 +342,7 @@ CoreEntry::CoreEntry() fCPUCount(0), fCPUIdleCount(0), fStarvationCounter(0), + fStarvationCounterIdle(0), fThreadCount(0), fActiveTime(0), fLoad(0), @@ -380,8 +378,6 @@ CoreEntry::PushBack(ThreadData* thread, int32 priority) SCHEDULER_ENTER_FUNCTION(); fRunQueue.PushBack(thread, priority); - fThreadList.Insert(thread); - atomic_add(&fThreadCount, 1); } @@ -393,12 +389,13 @@ CoreEntry::Remove(ThreadData* thread) ASSERT(thread->IsEnqueued()); thread->SetDequeued(); - if (thread_is_idle_thread(thread->GetThread()) - || fThreadList.Head() == thread) { + + ASSERT(!thread_is_idle_thread(thread->GetThread())); + if (thread->GetEffectivePriority() == B_LOWEST_ACTIVE_PRIORITY + || thread->IsCPUBound()) { atomic_add(&fStarvationCounter, 1); } - if (thread->WentSleepCount() == 0) - fThreadList.Remove(thread); + fRunQueue.Remove(thread); atomic_add(&fThreadCount, -1); } @@ -548,8 +545,8 @@ CoreLoadHeap::Dump() CoreEntry* entry = PeekMinimum(); while (entry) { int32 key = GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(), - entry->GetLoad() / 10); + kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(), + entry->GetLoad() / 10, entry->ThreadCount()); RemoveMinimum(); sDebugCoreHeap.Insert(entry, key); @@ -647,7 +644,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) static int -dump_run_queue(int argc, char **argv) +dump_run_queue(int /* argc */, char** /* argv */) { int32 cpuCount = smp_get_num_cpus(); int32 coreCount = gCoreCount; @@ -665,9 +662,9 @@ dump_run_queue(int argc, char **argv) static int -dump_cpu_heap(int argc, char** argv) +dump_cpu_heap(int /* argc */, char** /* argv */) { - kprintf("core load\n"); + kprintf("core load threads\n"); gCoreLoadHeap.Dump(); kprintf("\n"); gCoreHighLoadHeap.Dump(); @@ -685,7 +682,7 @@ dump_cpu_heap(int argc, char** argv) static int -dump_idle_cores(int argc, char** argv) +dump_idle_cores(int /* argc */, char** /* argv */) { kprintf("Idle packages:\n"); IdlePackageList::ReverseIterator idleIterator diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 1def98d4e6..95f20bc9f9 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -150,6 +150,7 @@ public: void UpdateLoad(int32 delta); inline int32 StarvationCounter() const; + inline int32 StarvationCounterIdle() const; inline void CPUGoesIdle(CPUEntry* cpu); inline void CPUWakesUp(CPUEntry* cpu); @@ -174,7 +175,7 @@ private: spinlock fCPULock; int32 fStarvationCounter; - DoublyLinkedList fThreadList; + int32 fStarvationCounterIdle; int32 fThreadCount; ThreadRunQueue fRunQueue; @@ -387,6 +388,14 @@ CoreEntry::StarvationCounter() const } +inline int32 +CoreEntry::StarvationCounterIdle() const +{ + SCHEDULER_ENTER_FUNCTION(); + return fStarvationCounterIdle; +} + + /* PackageEntry::CoreGoesIdle and PackageEntry::CoreWakesUp have to be defined before CoreEntry::CPUGoesIdle and CoreEntry::CPUWakesUp. If they weren't GCC2 wouldn't inline them as, apparently, it doesn't do enough optimization @@ -437,8 +446,13 @@ PackageEntry::CoreWakesUp(CoreEntry* core) inline void CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) { - ASSERT(fCPUIdleCount < fCPUCount); + atomic_add(&fStarvationCounter, 1); + atomic_add(&fStarvationCounterIdle, 1); + if (gSingleCore) + return; + + ASSERT(fCPUIdleCount < fCPUCount); if (++fCPUIdleCount == fCPUCount) fPackage->CoreGoesIdle(this); } @@ -447,8 +461,10 @@ CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) inline void CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) { - ASSERT(fCPUIdleCount > 0); + if (gSingleCore) + return; + ASSERT(fCPUIdleCount > 0); if (fCPUIdleCount-- == fCPUCount) fPackage->CoreWakesUp(this); } diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 09670dd160..820912afd9 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -13,8 +13,6 @@ struct scheduler_mode_operations { const char* name; - bool avoid_boost; - bigtime_t base_quantum; bigtime_t minimal_quantum; bigtime_t quantum_multipliers[2]; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index af4b40aa1e..e7a2445293 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -19,6 +19,9 @@ ThreadData::_InitBase() fAdditionalPenalty = 0; fEffectivePriority = fThread->priority; + fReceivedPenalty = false; + fHasSlept = false; + fTimeLeft = 0; fStolenTime = 0; @@ -28,7 +31,8 @@ ThreadData::_InitBase() fWentSleep = 0; fWentSleepActive = 0; - fWentSleepCount = -1; + fWentSleepCount = 0; + fWentSleepCountIdle = 0; fEnqueued = false; } @@ -124,7 +128,25 @@ ThreadData::Dump() const additionalPenalty = fAdditionalPenalty % kMinimalPriority; kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", additionalPenalty, fAdditionalPenalty); - kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", fStolenTime); + kprintf("\teffective_priority:\t%" B_PRId32 "\n", GetEffectivePriority()); + + kprintf("\treceived_penalty:\t%s\n", fReceivedPenalty ? "true" : "false"); + kprintf("\thas_slept:\t\t%s\n", fHasSlept ? "true" : "false"); + + bigtime_t quantum = _GetBaseQuantum(); + if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { + int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount(); + threadCount = max_c(threadCount, 1); + + quantum + = std::min(gCurrentMode->maximum_latency / threadCount, quantum); + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + } + kprintf("\ttime_left:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n", + fTimeLeft, quantum); + + kprintf("\tstolen_time:\t\t%" B_PRId64 " us\n", fStolenTime); + kprintf("\tquantum_start:\t\t%" B_PRId64 " us\n", fQuantumStart); kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", fLoad / 10); kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", fWentSleep); kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 5c8d052120..fd28497f3d 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -44,10 +44,11 @@ public: inline int32 GetEffectivePriority() const; - inline void IncreasePenalty(); inline void CancelPenalty(); inline bool ShouldCancelPenalty() const; + inline bool IsCPUBound() const { return fAdditionalPenalty != 0; } + bool ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU); @@ -58,7 +59,6 @@ public: inline void GoesAway(); inline bigtime_t WentSleep() const { return fWentSleep; } inline bigtime_t WentSleepActive() const { return fWentSleepActive; } - inline bigtime_t WentSleepCount() const { return fWentSleepCount; } inline void PutBack(); inline void Enqueue(); @@ -83,6 +83,7 @@ public: static void ComputeQuantumLengths(); private: + inline void _IncreasePenalty(); inline int32 _GetPenalty() const; void _ComputeEffectivePriority() const; @@ -98,6 +99,7 @@ private: bigtime_t fWentSleep; bigtime_t fWentSleepActive; int32 fWentSleepCount; + int32 fWentSleepCountIdle; bool fEnqueued; @@ -105,6 +107,8 @@ private: int32 fPriorityPenalty; int32 fAdditionalPenalty; + bool fReceivedPenalty; + bool fHasSlept; mutable int32 fEffectivePriority; @@ -167,7 +171,7 @@ ThreadData::GetEffectivePriority() const inline void -ThreadData::IncreasePenalty() +ThreadData::_IncreasePenalty() { SCHEDULER_ENTER_FUNCTION(); @@ -178,6 +182,7 @@ ThreadData::IncreasePenalty() TRACE("increasing thread %ld penalty\n", fThread->id); + fReceivedPenalty = true; int32 oldPenalty = fPriorityPenalty++; ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); @@ -216,9 +221,16 @@ ThreadData::ShouldCancelPenalty() const if (fCore == NULL) return false; + if (system_time() - fWentSleep > gCurrentMode->minimal_quantum * 2) + return false; - return fCore->StarvationCounter() != fWentSleepCount - && system_time() - fWentSleep > gCurrentMode->base_quantum; + if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY + && !IsCPUBound()) { + if (fCore->StarvationCounter() != fWentSleepCount) + return true; + } + + return fCore->StarvationCounterIdle() != fWentSleepCountIdle; } @@ -238,10 +250,15 @@ ThreadData::GoesAway() { SCHEDULER_ENTER_FUNCTION(); + if (!fReceivedPenalty) + _IncreasePenalty(); + fHasSlept = true; + fLastInterruptTime = 0; fWentSleep = system_time(); fWentSleepCount = fCore->StarvationCounter(); + fWentSleepCountIdle = fCore->StarvationCounterIdle(); fWentSleepActive = fCore->GetActiveTime(); } @@ -253,7 +270,6 @@ ThreadData::PutBack() if (gTrackLoad) ComputeLoad(); - fWentSleepCount = -1; int32 priority = GetEffectivePriority(); @@ -285,7 +301,6 @@ ThreadData::Enqueue() if (gTrackLoad) ComputeLoad(); - fWentSleepCount = 0; int32 priority = GetEffectivePriority(); @@ -328,7 +343,7 @@ ThreadData::Dequeue() CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; - ASSERT(fWentSleepCount < 1); + fCore->Remove(this); ASSERT(!fEnqueued); return true; @@ -354,15 +369,25 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) } bigtime_t timeUsed = system_time() - fQuantumStart; - fTimeLeft -= timeUsed; + if (timeUsed > 0); + fTimeLeft -= timeUsed; fTimeLeft = std::max(fTimeLeft, bigtime_t(0)); // too little time left, it's better make the next quantum a bit longer - if (wasPreempted || fTimeLeft <= gCurrentMode->minimal_quantum) { + int32 skipTime = gCurrentMode->minimal_quantum; + skipTime -= skipTime / 10; + if (wasPreempted || fTimeLeft <= skipTime) { fStolenTime += fTimeLeft; fTimeLeft = 0; } + if (fTimeLeft == 0) { + if (!fReceivedPenalty && !fHasSlept) + _IncreasePenalty(); + fReceivedPenalty = false; + fHasSlept = false; + } + return fTimeLeft == 0; } From d36098e0430bdec4c5202673c3a8bff776dd03db Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 8 Jan 2014 05:02:04 +0100 Subject: [PATCH 262/273] scheduler: Keep track of the number of the ready threads --- src/system/kernel/scheduler/scheduler.cpp | 15 +------- src/system/kernel/scheduler/scheduler_cpu.cpp | 35 +++++++++++++++---- src/system/kernel/scheduler/scheduler_cpu.h | 12 ++++--- .../kernel/scheduler/scheduler_thread.cpp | 8 +++++ .../kernel/scheduler/scheduler_thread.h | 17 +++++++++ 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 0ee4cd0bd8..babf25b872 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -53,20 +53,6 @@ bool gSingleCore; bool gCPUFrequencyManagement; bool gTrackLoad; -CPUEntry* gCPUEntries; - -CoreEntry* gCoreEntries; -CoreLoadHeap gCoreLoadHeap; -CoreLoadHeap gCoreHighLoadHeap; -rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; -int32 gCoreCount; - -PackageEntry* gPackageEntries; -IdlePackageList gIdlePackageList; -rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; -int32 gPackageCount; - - } // namespace Scheduler using namespace Scheduler; @@ -415,6 +401,7 @@ reschedule(int32 nextState) break; case THREAD_STATE_FREE_ON_RESCHED: + oldThreadData->Dies(); break; default: oldThreadData->GoesAway(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 4bdb566840..d715e72d44 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -13,6 +13,25 @@ #include "scheduler_thread.h" +namespace Scheduler { + + +CPUEntry* gCPUEntries; + +CoreEntry* gCoreEntries; +CoreLoadHeap gCoreLoadHeap; +CoreLoadHeap gCoreHighLoadHeap; +rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; +int32 gCoreCount; + +PackageEntry* gPackageEntries; +IdlePackageList gIdlePackageList; +rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; +int32 gPackageCount; + + +} // namespace Scheduler + using namespace Scheduler; @@ -340,7 +359,7 @@ CPUPriorityHeap::Dump() CoreEntry::CoreEntry() : fCPUCount(0), - fCPUIdleCount(0), + fIdleCPUCount(0), fStarvationCounter(0), fStarvationCounterIdle(0), fThreadCount(0), @@ -459,9 +478,9 @@ void CoreEntry::AddCPU(CPUEntry* cpu) { ASSERT(fCPUCount >= 0); - ASSERT(fCPUIdleCount >= 0); + ASSERT(fIdleCPUCount >= 0); - fCPUIdleCount++; + fIdleCPUCount++; if (fCPUCount++ == 0) { // core has been reenabled fLoad = 0; @@ -479,9 +498,9 @@ void CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) { ASSERT(fCPUCount > 0); - ASSERT(fCPUIdleCount > 0); + ASSERT(fIdleCPUCount > 0); - fCPUIdleCount--; + fIdleCPUCount--; if (--fCPUCount == 0) { // core has been disabled if (fHighLoad) { @@ -545,8 +564,10 @@ CoreLoadHeap::Dump() CoreEntry* entry = PeekMinimum(); while (entry) { int32 key = GetKey(entry); + + int32 activeCPUs = entry->CPUCount() - entry->IdleCPUCount(); kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(), - entry->GetLoad() / 10, entry->ThreadCount()); + entry->GetLoad() / 10, entry->ThreadCount() + activeCPUs); RemoveMinimum(); sDebugCoreHeap.Insert(entry, key); @@ -664,6 +685,8 @@ dump_run_queue(int /* argc */, char** /* argv */) static int dump_cpu_heap(int /* argc */, char** /* argv */) { + kprintf("Total ready threads: %" B_PRId32 "\n\n", gReadyThreadCount); + kprintf("core load threads\n"); gCoreLoadHeap.Dump(); kprintf("\n"); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 95f20bc9f9..6c1d209971 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -123,6 +123,8 @@ public: inline PackageEntry* Package() const { return fPackage; } inline int32 CPUCount() const { return fCPUCount; } + inline int32 IdleCPUCount() const + { return fIdleCPUCount; } inline void LockCPUHeap(); inline void UnlockCPUHeap(); @@ -170,7 +172,7 @@ private: PackageEntry* fPackage; int32 fCPUCount; - int32 fCPUIdleCount; + int32 fIdleCPUCount; CPUPriorityHeap fCPUHeap; spinlock fCPULock; @@ -452,8 +454,8 @@ CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) if (gSingleCore) return; - ASSERT(fCPUIdleCount < fCPUCount); - if (++fCPUIdleCount == fCPUCount) + ASSERT(fIdleCPUCount < fCPUCount); + if (++fIdleCPUCount == fCPUCount) fPackage->CoreGoesIdle(this); } @@ -464,8 +466,8 @@ CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) if (gSingleCore) return; - ASSERT(fCPUIdleCount > 0); - if (fCPUIdleCount-- == fCPUCount) + ASSERT(fIdleCPUCount > 0); + if (fIdleCPUCount-- == fCPUCount) fPackage->CoreWakesUp(this); } diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index e7a2445293..63f4450b6e 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -6,6 +6,14 @@ #include "scheduler_thread.h" +namespace Scheduler { + + +int32 gReadyThreadCount; + + +} // namespace Scheduler + using namespace Scheduler; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index fd28497f3d..896692095b 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -57,6 +57,8 @@ public: inline void SetStolenInterruptTime(bigtime_t interruptTime); inline void GoesAway(); + inline void Dies(); + inline bigtime_t WentSleep() const { return fWentSleep; } inline bigtime_t WentSleepActive() const { return fWentSleepActive; } @@ -128,6 +130,8 @@ public: virtual void operator()(ThreadData* thread) = 0; }; +extern int32 gReadyThreadCount; + inline int32 ThreadData::_GetMinimalPriority() const @@ -260,6 +264,16 @@ ThreadData::GoesAway() fWentSleepCount = fCore->StarvationCounter(); fWentSleepCountIdle = fCore->StarvationCounterIdle(); fWentSleepActive = fCore->GetActiveTime(); + + atomic_add(&gReadyThreadCount, -1); +} + + +inline void +ThreadData::Dies() +{ + SCHEDULER_ENTER_FUNCTION(); + atomic_add(&gReadyThreadCount, -1); } @@ -297,6 +311,9 @@ ThreadData::Enqueue() { SCHEDULER_ENTER_FUNCTION(); + if (fThread->state != B_THREAD_READY && fThread->state != B_THREAD_RUNNING) + atomic_add(&gReadyThreadCount, 1); + fThread->state = B_THREAD_READY; if (gTrackLoad) From 772331c7cdd486b7283ea138621693df88a9327b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 8 Jan 2014 07:03:22 +0100 Subject: [PATCH 263/273] scheduler: Introduce strong and weak priority penalties --- src/system/kernel/scheduler/low_latency.cpp | 2 +- .../kernel/scheduler/scheduler_thread.cpp | 12 ++++----- .../kernel/scheduler/scheduler_thread.h | 26 +++++++++++++------ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 51381c3ac8..3b567cbae5 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -164,7 +164,7 @@ scheduler_mode_operations gSchedulerLowLatencyMode = { 100, { 2, 25 }, - 50000, + 10000, switch_to_mode, set_cpu_enabled, diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 63f4450b6e..a16cf3259c 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -217,14 +217,14 @@ ThreadData::ComputeQuantum() quantum += fStolenTime; fStolenTime = 0; - int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount(); - threadCount = max_c(threadCount, 1); - - quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum); - quantum = std::max(quantum, gCurrentMode->minimal_quantum); + int32 threadCount = fCore->ThreadCount() / fCore->CPUCount(); + if (threadCount >= 1) { + quantum + = std::min(gCurrentMode->maximum_latency / threadCount, quantum); + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + } fTimeLeft = quantum; - return quantum; } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 896692095b..92e5f2353a 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -44,11 +44,11 @@ public: inline int32 GetEffectivePriority() const; + inline bool IsCPUBound() const; + inline void CancelPenalty(); inline bool ShouldCancelPenalty() const; - inline bool IsCPUBound() const { return fAdditionalPenalty != 0; } - bool ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU); @@ -85,7 +85,7 @@ public: static void ComputeQuantumLengths(); private: - inline void _IncreasePenalty(); + inline void _IncreasePenalty(bool strong); inline int32 _GetPenalty() const; void _ComputeEffectivePriority() const; @@ -175,7 +175,7 @@ ThreadData::GetEffectivePriority() const inline void -ThreadData::_IncreasePenalty() +ThreadData::_IncreasePenalty(bool strong) { SCHEDULER_ENTER_FUNCTION(); @@ -187,12 +187,14 @@ ThreadData::_IncreasePenalty() TRACE("increasing thread %ld penalty\n", fThread->id); fReceivedPenalty = true; - int32 oldPenalty = fPriorityPenalty++; + int32 oldPenalty = fPriorityPenalty; + if (strong) + fPriorityPenalty++; ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); const int kMinimalPriority = _GetMinimalPriority(); - if (fThread->priority - oldPenalty <= kMinimalPriority) { + if (!strong || fThread->priority - oldPenalty <= kMinimalPriority) { fPriorityPenalty = oldPenalty; fAdditionalPenalty++; } @@ -201,6 +203,14 @@ ThreadData::_IncreasePenalty() } +inline bool +ThreadData::IsCPUBound() const +{ + SCHEDULER_ENTER_FUNCTION(); + return fAdditionalPenalty != 0 && fPriorityPenalty != 0; +} + + inline void ThreadData::CancelPenalty() { @@ -255,7 +265,7 @@ ThreadData::GoesAway() SCHEDULER_ENTER_FUNCTION(); if (!fReceivedPenalty) - _IncreasePenalty(); + _IncreasePenalty(false); fHasSlept = true; fLastInterruptTime = 0; @@ -400,7 +410,7 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) if (fTimeLeft == 0) { if (!fReceivedPenalty && !fHasSlept) - _IncreasePenalty(); + _IncreasePenalty(true); fReceivedPenalty = false; fHasSlept = false; } From a2634874ed5e33a36fe83c272614e2042fafde1d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 8 Jan 2014 22:59:04 +0100 Subject: [PATCH 264/273] scheduler: Estimate the load thread is able to produce Previous implementation based on the actual load of each core and share each thread has in that load turned up to be very problematic when balancing load on very heavily loaded systems (i.e. more threads consuming all available CPU time than there is logical CPUs). The new approach is to estimate how much load would a thread produce if it had all CPU time only for itself. Summing such load estimations of each thread assigned to a given core we get a rank that contains much more information than just simple actual core load. --- headers/private/kernel/load_tracking.h | 5 +- src/system/kernel/int.cpp | 12 ++- src/system/kernel/scheduler/low_latency.cpp | 12 +-- src/system/kernel/scheduler/power_saving.cpp | 2 +- src/system/kernel/scheduler/scheduler.cpp | 47 ++++++---- .../kernel/scheduler/scheduler_common.h | 4 +- src/system/kernel/scheduler/scheduler_cpu.cpp | 50 +++++----- src/system/kernel/scheduler/scheduler_cpu.h | 14 ++- .../kernel/scheduler/scheduler_thread.cpp | 58 +++++++----- .../kernel/scheduler/scheduler_thread.h | 94 ++++++++++++++----- 10 files changed, 186 insertions(+), 112 deletions(-) diff --git a/headers/private/kernel/load_tracking.h b/headers/private/kernel/load_tracking.h index 934bcc79de..b4ff87a046 100644 --- a/headers/private/kernel/load_tracking.h +++ b/headers/private/kernel/load_tracking.h @@ -15,10 +15,9 @@ const bigtime_t kIntervalInaccuracy = kLoadMeasureInterval / 4; static inline int32 -compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load) +compute_load(bigtime_t& measureTime, bigtime_t& measureActiveTime, int32& load, + bigtime_t now) { - bigtime_t now = system_time(); - if (measureTime == 0) { measureTime = now; return -1; diff --git a/src/system/kernel/int.cpp b/src/system/kernel/int.cpp index c09d8bc6c3..b34ba01c7f 100644 --- a/src/system/kernel/int.cpp +++ b/src/system/kernel/int.cpp @@ -256,7 +256,7 @@ update_int_load(int i) int32 oldLoad = sVectors[i].load; compute_load(sVectors[i].last_measure_time, sVectors[i].last_measure_active, - sVectors[i].load); + sVectors[i].load, system_time()); if (oldLoad != sVectors[i].load) atomic_add(&sVectors[i].assigned_cpu->load, sVectors[i].load - oldLoad); @@ -365,9 +365,13 @@ int_io_interrupt_handler(int vector, bool levelTriggered) vectorLocker.Unlock(); cpu_ent* cpu = get_cpu_struct(); - cpu->interrupt_time += deltaTime; - if (sVectors[vector].type == INTERRUPT_TYPE_IRQ) - cpu->irq_time += deltaTime; + if (sVectors[vector].type == INTERRUPT_TYPE_IRQ + || sVectors[vector].type == INTERRUPT_TYPE_ICI + || sVectors[vector].type == INTERRUPT_TYPE_LOCAL_IRQ) { + cpu->interrupt_time += deltaTime; + if (sVectors[vector].type == INTERRUPT_TYPE_IRQ) + cpu->irq_time += deltaTime; + } update_int_load(vector); diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 3b567cbae5..2d6b70990c 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -78,13 +78,13 @@ should_rebalance(const ThreadData* threadData) SCHEDULER_ENTER_FUNCTION(); int32 coreLoad = threadData->Core()->GetLoad(); + int32 threadLoad = threadData->GetLoad() / threadData->Core()->CPUCount(); // If the thread produces more than 50% of the load, leave it here. In // such situation it is better to move other threads away. - if (threadData->GetLoad() >= coreLoad / 2) + if (threadLoad >= coreLoad / 2) return false; - int32 threadLoad = threadData->GetLoad(); int32 coreNewLoad = coreLoad - threadLoad; // If there is high load on this core but this thread does not contribute @@ -160,11 +160,11 @@ rebalance_irqs(bool idle) scheduler_mode_operations gSchedulerLowLatencyMode = { "low latency", - 2000, - 100, - { 2, 25 }, + 1000, + 50, + { 2, 50 }, - 10000, + 50000, switch_to_mode, set_cpu_enabled, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 66efc8ce9b..7c19c297eb 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -123,7 +123,7 @@ should_rebalance(const ThreadData* threadData) CoreEntry* core = threadData->Core(); int32 coreLoad = core->GetLoad(); - int32 threadLoad = threadData->GetLoad(); + int32 threadLoad = threadData->GetLoad() / core->CPUCount(); if (coreLoad > kHighLoad) { if (sSmallTaskCore == core) { sSmallTaskCore = NULL; diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index babf25b872..8b32b5c227 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -50,8 +50,8 @@ scheduler_mode gCurrentModeID; scheduler_mode_operations* gCurrentMode; bool gSingleCore; -bool gCPUFrequencyManagement; -bool gTrackLoad; +bool gTrackCoreLoad; +bool gTrackCPULoad; } // namespace Scheduler @@ -388,23 +388,33 @@ reschedule(int32 nextState) case B_THREAD_READY: enqueueOldThread = true; - if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, - oldThread->has_yielded)) { - TRACE("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, oldThreadData->GetEffectivePriority()); - putOldThreadAtBack = true; - } else { - TRACE("putting thread %ld back in run queue priority = %ld\n", - oldThread->id, oldThreadData->GetEffectivePriority()); - putOldThreadAtBack = false; + if (!thread_is_idle_thread(oldThread)) { + oldThreadData->Continues(); + if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, + oldThread->has_yielded)) { + TRACE("enqueueing thread %ld into run queue priority =" + " %ld\n", oldThread->id, + oldThreadData->GetEffectivePriority()); + putOldThreadAtBack = true; + } else { + TRACE("putting thread %ld back in run queue priority =" + " %ld\n", oldThread->id, + oldThreadData->GetEffectivePriority()); + putOldThreadAtBack = false; + } } break; case THREAD_STATE_FREE_ON_RESCHED: oldThreadData->Dies(); + if (gCPU[thisCPU].disabled) + oldThreadData->UnassignCore(true); break; default: oldThreadData->GoesAway(); + if (gCPU[thisCPU].disabled) + oldThreadData->UnassignCore(true); + TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, nextState); break; @@ -422,6 +432,7 @@ reschedule(int32 nextState) cpu->Remove(nextThreadData); putOldThreadAtBack = oldThread->pinned_to_cpu == 0; + oldThreadData->UnassignCore(true); } else nextThreadData = oldThreadData; } else { @@ -476,6 +487,8 @@ reschedule(int32 nextState) bigtime_t quantum = nextThreadData->ComputeQuantum(); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); + + nextThreadData->Continues(); } else gCurrentMode->rebalance_irqs(true); nextThreadData->StartQuantum(); @@ -696,12 +709,12 @@ init() // disable parts of the scheduler logic that are not needed gSingleCore = coreCount == 1; - gCPUFrequencyManagement = increase_cpu_performance(0) == B_OK; - gTrackLoad = !gSingleCore || gCPUFrequencyManagement; - dprintf("scheduler switches: single core: %s, cpufreq: %s, load tracking:" - " %s\n", gSingleCore ? "true" : "false", - gCPUFrequencyManagement ? "true" : "false", - gTrackLoad ? "true" : "false"); + gTrackCPULoad = increase_cpu_performance(0) == B_OK; + gTrackCoreLoad = !gSingleCore || gTrackCPULoad; + dprintf("scheduler switches: single core: %s, cpu load tracking: %s," + " core load tracking: %s\n", gSingleCore ? "true" : "false", + gTrackCPULoad ? "true" : "false", + gTrackCoreLoad ? "true" : "false"); gCoreCount = coreCount; gPackageCount = packageCount; diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 812fb7d8d0..96e7d867ce 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -43,8 +43,8 @@ const int kVeryHighLoad = (kMaxLoad + kHighLoad) / 2; const int kLoadDifference = kMaxLoad * 20 / 100; extern bool gSingleCore; -extern bool gCPUFrequencyManagement; -extern bool gTrackLoad; +extern bool gTrackCoreLoad; +extern bool gTrackCPULoad; void init_debug_commands(); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index d715e72d44..9cbee92710 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -190,19 +190,15 @@ CPUEntry::ComputeLoad() { SCHEDULER_ENTER_FUNCTION(); - ASSERT(gTrackLoad); + ASSERT(gTrackCPULoad); ASSERT(!gCPU[fCPUNumber].disabled); ASSERT(fCPUNumber == smp_get_current_cpu()); - int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad); + int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad, + system_time()); if (oldLoad < 0) return; - if (oldLoad != fLoad) { - int32 delta = fLoad - oldLoad; - fCore->UpdateLoad(delta); - } - if (fLoad > kVeryHighLoad) gCurrentMode->rebalance_irqs(false); } @@ -272,11 +268,10 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) oldThreadData->UpdateActivity(active); } - if (gTrackLoad) { - oldThreadData->ComputeLoad(); - nextThreadData->ComputeLoad(); + if (gTrackCPULoad) { if (!cpuEntry->disabled) ComputeLoad(); + _RequestPerformanceLevel(nextThreadData); } Thread* nextThread = nextThreadData->GetThread(); @@ -285,9 +280,6 @@ CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) cpuEntry->last_user_time = nextThread->user_time; nextThreadData->SetLastInterruptTime(cpuEntry->interrupt_time); - - if (gCPUFrequencyManagement) - _RequestPerformanceLevel(nextThreadData); } } @@ -365,7 +357,8 @@ CoreEntry::CoreEntry() fThreadCount(0), fActiveTime(0), fLoad(0), - fHighLoad(false) + fHighLoad(false), + fLastLoadUpdate(0) { B_INITIALIZE_SPINLOCK(&fCPULock); B_INITIALIZE_SPINLOCK(&fQueueLock); @@ -425,20 +418,24 @@ CoreEntry::UpdateLoad(int32 delta) { SCHEDULER_ENTER_FUNCTION(); - if (fCPUCount == 0) { - fLoad = 0; - return; - } + ASSERT(gTrackCoreLoad); atomic_add(&fLoad, delta); - WriteSpinLocker coreLocker(gCoreHeapsLock); + bigtime_t now = system_time(); + if (now < kLoadMeasureInterval + fLastLoadUpdate) + return; + if (!try_acquire_write_spinlock(&gCoreHeapsLock)) + return; + WriteSpinLocker coreLocker(gCoreHeapsLock, true); + + fLastLoadUpdate = now; int32 newKey = GetLoad(); int32 oldKey = CoreLoadHeap::GetKey(this); - ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); - ASSERT(newKey >= 0 && newKey <= kMaxLoad); + ASSERT(oldKey >= 0); + ASSERT(newKey >= 0); if (oldKey == newKey) return; @@ -502,6 +499,9 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) fIdleCPUCount--; if (--fCPUCount == 0) { + // unassign threads + thread_map(CoreEntry::_UnassignThread, this); + // core has been disabled if (fHighLoad) { gCoreHighLoadHeap.ModifyKey(this, -1); @@ -516,8 +516,6 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) fPackage->RemoveIdleCore(this); // get rid of threads - thread_map(CoreEntry::_UnassignThread, this); - while (fRunQueue.PeekMaximum() != NULL) { ThreadData* threadData = fRunQueue.PeekMaximum(); @@ -535,7 +533,6 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) fCPUHeap.RemoveRoot(); ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad); - fLoad -= cpu->GetLoad(); ASSERT(fLoad >= 0); } @@ -565,9 +562,8 @@ CoreLoadHeap::Dump() while (entry) { int32 key = GetKey(entry); - int32 activeCPUs = entry->CPUCount() - entry->IdleCPUCount(); kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(), - entry->GetLoad() / 10, entry->ThreadCount() + activeCPUs); + entry->GetLoad() / 10, entry->ThreadCount()); RemoveMinimum(); sDebugCoreHeap.Insert(entry, key); @@ -685,8 +681,6 @@ dump_run_queue(int /* argc */, char** /* argv */) static int dump_cpu_heap(int /* argc */, char** /* argv */) { - kprintf("Total ready threads: %" B_PRId32 "\n\n", gReadyThreadCount); - kprintf("core load threads\n"); gCoreLoadHeap.Dump(); kprintf("\n"); diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 6c1d209971..f8664e7f65 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -123,16 +123,13 @@ public: inline PackageEntry* Package() const { return fPackage; } inline int32 CPUCount() const { return fCPUCount; } - inline int32 IdleCPUCount() const - { return fIdleCPUCount; } inline void LockCPUHeap(); inline void UnlockCPUHeap(); inline CPUPriorityHeap* CPUHeap(); - inline int32 ThreadCount() const - { return fThreadCount; } + inline int32 ThreadCount() const; inline void LockRunQueue(); inline void UnlockRunQueue(); @@ -188,6 +185,7 @@ private: int32 fLoad; bool fHighLoad; + bigtime_t fLastLoadUpdate; friend class DebugDumper; } CACHE_LINE_ALIGN; @@ -332,6 +330,14 @@ CoreEntry::CPUHeap() } +inline int32 +CoreEntry::ThreadCount() const +{ + SCHEDULER_ENTER_FUNCTION(); + return fThreadCount + fCPUCount - fIdleCPUCount; +} + + inline void CoreEntry::LockRunQueue() { diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index a16cf3259c..f3c4983c2a 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -6,14 +6,6 @@ #include "scheduler_thread.h" -namespace Scheduler { - - -int32 gReadyThreadCount; - - -} // namespace Scheduler - using namespace Scheduler; @@ -33,9 +25,11 @@ ThreadData::_InitBase() fTimeLeft = 0; fStolenTime = 0; - fMeasureActiveTime = 0; - fMeasureTime = 0; - fLoad = 0; + fMeasureAvailableActiveTime = 0; + fLastMeasureAvailableTime = 0; + fMeasureAvailableTime = 0; + + fNeededLoad = 0; fWentSleep = 0; fWentSleepActive = 0; @@ -43,6 +37,7 @@ ThreadData::_InitBase() fWentSleepCountIdle = 0; fEnqueued = false; + fReady = false; } @@ -113,6 +108,14 @@ ThreadData::Init() ThreadData* currentThreadData = thread_get_current_thread()->scheduler_data; fCore = currentThreadData->fCore; + + if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { + fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty, + std::max(fThread->priority - _GetMinimalPriority(), int32(0))); + fAdditionalPenalty = currentThreadData->fAdditionalPenalty; + + _ComputeEffectivePriority(); + } } @@ -122,6 +125,7 @@ ThreadData::Init(CoreEntry* core) _InitBase(); fCore = core; + fReady = true; } @@ -155,7 +159,7 @@ ThreadData::Dump() const kprintf("\tstolen_time:\t\t%" B_PRId64 " us\n", fStolenTime); kprintf("\tquantum_start:\t\t%" B_PRId64 " us\n", fQuantumStart); - kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", fLoad / 10); + kprintf("\tneeded_load:\t\t%" B_PRId32 "%%\n", fNeededLoad / 10); kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", fWentSleep); kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount); @@ -185,21 +189,16 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) ASSERT(targetCore != NULL); ASSERT(targetCPU != NULL); + if (fReady && fCore != targetCore && fCore != NULL) { + fCore->UpdateLoad(-fNeededLoad); + targetCore->UpdateLoad(fNeededLoad); + } + fCore = targetCore; return rescheduleNeeded; } -void -ThreadData::ComputeLoad() -{ - SCHEDULER_ENTER_FUNCTION(); - - ASSERT(gTrackLoad); - compute_load(fMeasureTime, fMeasureActiveTime, fLoad); -} - - bigtime_t ThreadData::ComputeQuantum() { @@ -272,6 +271,21 @@ ThreadData::_GetPenalty() const } +void +ThreadData::_ComputeNeededLoad() +{ + SCHEDULER_ENTER_FUNCTION(); + + int32 oldLoad = compute_load(fLastMeasureAvailableTime, + fMeasureAvailableActiveTime, fNeededLoad, fMeasureAvailableTime); + if (oldLoad < 0 || oldLoad == fNeededLoad) + return; + + int32 delta = fNeededLoad - oldLoad; + fCore->UpdateLoad(delta); +} + + void ThreadData::_ComputeEffectivePriority() const { diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 92e5f2353a..5d2af64849 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -56,6 +56,7 @@ public: { fLastInterruptTime = interruptTime; } inline void SetStolenInterruptTime(bigtime_t interruptTime); + inline void Continues(); inline void GoesAway(); inline void Dies(); @@ -67,7 +68,6 @@ public: inline bool Dequeue(); inline void UpdateActivity(bigtime_t active); - void ComputeLoad(); inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); bigtime_t ComputeQuantum(); @@ -77,10 +77,10 @@ public: inline void SetDequeued() { fEnqueued = false; } inline Thread* GetThread() const { return fThread; } - inline int32 GetLoad() const { return fLoad; } + inline int32 GetLoad() const { return fNeededLoad; } inline CoreEntry* Core() const { return fCore; } - inline void UnassignCore() { fCore = NULL; } + inline void UnassignCore(bool running = false); static void ComputeQuantumLengths(); @@ -88,6 +88,8 @@ private: inline void _IncreasePenalty(bool strong); inline int32 _GetPenalty() const; + void _ComputeNeededLoad(); + void _ComputeEffectivePriority() const; static bigtime_t _ScaleQuantum(bigtime_t maxQuantum, @@ -104,6 +106,7 @@ private: int32 fWentSleepCountIdle; bool fEnqueued; + bool fReady; Thread* fThread; @@ -116,9 +119,11 @@ private: bigtime_t fTimeLeft; - bigtime_t fMeasureActiveTime; - bigtime_t fMeasureTime; - int32 fLoad; + bigtime_t fMeasureAvailableActiveTime; + bigtime_t fMeasureAvailableTime; + bigtime_t fLastMeasureAvailableTime; + + int32 fNeededLoad; CoreEntry* fCore; }; @@ -130,8 +135,6 @@ public: virtual void operator()(ThreadData* thread) = 0; }; -extern int32 gReadyThreadCount; - inline int32 ThreadData::_GetMinimalPriority() const @@ -235,8 +238,6 @@ ThreadData::ShouldCancelPenalty() const if (fCore == NULL) return false; - if (system_time() - fWentSleep > gCurrentMode->minimal_quantum * 2) - return false; if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY && !IsCPUBound()) { @@ -255,7 +256,17 @@ ThreadData::SetStolenInterruptTime(bigtime_t interruptTime) interruptTime -= fLastInterruptTime; fStolenTime += interruptTime; - fMeasureActiveTime -= interruptTime; +} + + +inline void +ThreadData::Continues() +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(fReady); + if (gTrackCoreLoad) + _ComputeNeededLoad(); } @@ -264,6 +275,8 @@ ThreadData::GoesAway() { SCHEDULER_ENTER_FUNCTION(); + ASSERT(fReady); + if (!fReceivedPenalty) _IncreasePenalty(false); fHasSlept = true; @@ -275,7 +288,9 @@ ThreadData::GoesAway() fWentSleepCountIdle = fCore->StarvationCounterIdle(); fWentSleepActive = fCore->GetActiveTime(); - atomic_add(&gReadyThreadCount, -1); + if (gTrackCoreLoad) + fCore->UpdateLoad(-fNeededLoad); + fReady = false; } @@ -283,7 +298,11 @@ inline void ThreadData::Dies() { SCHEDULER_ENTER_FUNCTION(); - atomic_add(&gReadyThreadCount, -1); + + ASSERT(fReady); + if (gTrackCoreLoad) + fCore->UpdateLoad(-fNeededLoad); + fReady = false; } @@ -292,9 +311,6 @@ ThreadData::PutBack() { SCHEDULER_ENTER_FUNCTION(); - if (gTrackLoad) - ComputeLoad(); - int32 priority = GetEffectivePriority(); if (fThread->pinned_to_cpu > 0) { @@ -321,14 +337,19 @@ ThreadData::Enqueue() { SCHEDULER_ENTER_FUNCTION(); - if (fThread->state != B_THREAD_READY && fThread->state != B_THREAD_RUNNING) - atomic_add(&gReadyThreadCount, 1); + if (!fReady) { + ASSERT(system_time() - fWentSleep > 0); + if (gTrackCoreLoad) { + fMeasureAvailableTime += system_time() - fWentSleep; + + fCore->UpdateLoad(fNeededLoad); + _ComputeNeededLoad(); + } + fReady = true; + } fThread->state = B_THREAD_READY; - if (gTrackLoad) - ComputeLoad(); - int32 priority = GetEffectivePriority(); if (fThread->pinned_to_cpu > 0) { @@ -381,7 +402,12 @@ inline void ThreadData::UpdateActivity(bigtime_t active) { SCHEDULER_ENTER_FUNCTION(); - fMeasureActiveTime += active; + + if (!gTrackCoreLoad) + return; + + fMeasureAvailableTime += active; + fMeasureAvailableActiveTime += active; } @@ -396,9 +422,8 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) } bigtime_t timeUsed = system_time() - fQuantumStart; - if (timeUsed > 0); - fTimeLeft -= timeUsed; - fTimeLeft = std::max(fTimeLeft, bigtime_t(0)); + ASSERT(timeUsed >= 0); + fTimeLeft -= timeUsed; // too little time left, it's better make the next quantum a bit longer int32 skipTime = gCurrentMode->minimal_quantum; @@ -427,6 +452,25 @@ ThreadData::StartQuantum() } +inline void +ThreadData::UnassignCore(bool running) +{ + SCHEDULER_ENTER_FUNCTION(); + + ASSERT(fCore != NULL); + if (!fReady) + fCore = NULL; + + if (running || fThread->state == B_THREAD_READY) { + if (gTrackCoreLoad) + fCore->UpdateLoad(-fNeededLoad); + fReady = false; + fThread->state = B_THREAD_SUSPENDED; + fCore = NULL; + } +} + + } // namespace Scheduler From 082d3c1015c3610e190fda886ea207c122181dc3 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 9 Jan 2014 03:48:48 +0100 Subject: [PATCH 265/273] scheduler: Increase thread penalty at fork --- src/system/kernel/scheduler/scheduler_thread.cpp | 8 +++++++- src/system/kernel/scheduler/scheduler_thread.h | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index f3c4983c2a..0ca3e50ea4 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -106,10 +106,16 @@ ThreadData::Init() { _InitBase(); - ThreadData* currentThreadData = thread_get_current_thread()->scheduler_data; + Thread* currentThread = thread_get_current_thread(); + ThreadData* currentThreadData = currentThread->scheduler_data; fCore = currentThreadData->fCore; if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { + if (!thread_is_idle_thread(currentThread) + && currentThread->priority < B_FIRST_REAL_TIME_PRIORITY) { + currentThreadData->_IncreasePenalty(false); + } + fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty, std::max(fThread->priority - _GetMinimalPriority(), int32(0))); fAdditionalPenalty = currentThreadData->fAdditionalPenalty; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 5d2af64849..26b1e43ba2 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -426,8 +426,7 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) fTimeLeft -= timeUsed; // too little time left, it's better make the next quantum a bit longer - int32 skipTime = gCurrentMode->minimal_quantum; - skipTime -= skipTime / 10; + int32 skipTime = gCurrentMode->minimal_quantum / 2; if (wasPreempted || fTimeLeft <= skipTime) { fStolenTime += fTimeLeft; fTimeLeft = 0; From 7f212f45c3d669962ca86d14a3b6d80246b1a486 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Wed, 15 Jan 2014 00:44:32 +0100 Subject: [PATCH 266/273] scheduler: Update used time when thread yields or sleeps --- src/system/kernel/scheduler/scheduler_thread.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 26b1e43ba2..f3f7020781 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -277,6 +277,10 @@ ThreadData::GoesAway() ASSERT(fReady); + bigtime_t timeUsed = system_time() - fQuantumStart; + ASSERT(timeUsed >= 0); + fTimeLeft -= timeUsed; + if (!fReceivedPenalty) _IncreasePenalty(false); fHasSlept = true; @@ -416,15 +420,15 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) { SCHEDULER_ENTER_FUNCTION(); + bigtime_t timeUsed = system_time() - fQuantumStart; + ASSERT(timeUsed >= 0); + fTimeLeft -= timeUsed; + if (hasYielded) { fTimeLeft = 0; return true; } - bigtime_t timeUsed = system_time() - fQuantumStart; - ASSERT(timeUsed >= 0); - fTimeLeft -= timeUsed; - // too little time left, it's better make the next quantum a bit longer int32 skipTime = gCurrentMode->minimal_quantum / 2; if (wasPreempted || fTimeLeft <= skipTime) { From 093c2202675b2ef2c9a76dec558fe6ed4a5e6f17 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 16 Jan 2014 22:32:35 +0100 Subject: [PATCH 267/273] scheduler: Improve latencies --- src/system/kernel/scheduler/low_latency.cpp | 6 +- src/system/kernel/scheduler/power_saving.cpp | 8 +- src/system/kernel/scheduler/scheduler.cpp | 4 +- .../kernel/scheduler/scheduler_thread.cpp | 51 +------ .../kernel/scheduler/scheduler_thread.h | 144 +++++++++--------- 5 files changed, 89 insertions(+), 124 deletions(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 2d6b70990c..20555e582c 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -161,10 +161,10 @@ scheduler_mode_operations gSchedulerLowLatencyMode = { "low latency", 1000, - 50, - { 2, 50 }, + 100, + { 2, 5 }, - 50000, + 5000, switch_to_mode, set_cpu_enabled, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 7c19c297eb..54c2db1795 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -236,11 +236,11 @@ rebalance_irqs(bool idle) scheduler_mode_operations gSchedulerPowerSavingMode = { "power saving", - 3000, - 1000, - { 3, 60 }, + 2000, + 500, + { 3, 10 }, - 200000, + 20000, switch_to_mode, set_cpu_enabled, diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 8b32b5c227..557344dc73 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Copyright 2013-2014, Paweł Dziepak, pdziepak@quarnos.org. * Copyright 2009, Rene Gollent, rene@gollent.com. * Copyright 2008-2011, Ingo Weinhold, ingo_weinhold@gmx.de. * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de. @@ -484,7 +484,7 @@ reschedule(int32 nextState) oldThread->cpu->preempted = false; if (!thread_is_idle_thread(nextThread)) { - bigtime_t quantum = nextThreadData->ComputeQuantum(); + bigtime_t quantum = nextThreadData->GetQuantumLeft(); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 0ca3e50ea4..ca6fada228 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -19,10 +19,7 @@ ThreadData::_InitBase() fAdditionalPenalty = 0; fEffectivePriority = fThread->priority; - fReceivedPenalty = false; - fHasSlept = false; - - fTimeLeft = 0; + fTimeUsed = 0; fStolenTime = 0; fMeasureAvailableActiveTime = 0; @@ -111,11 +108,6 @@ ThreadData::Init() fCore = currentThreadData->fCore; if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { - if (!thread_is_idle_thread(currentThread) - && currentThread->priority < B_FIRST_REAL_TIME_PRIORITY) { - currentThreadData->_IncreasePenalty(false); - } - fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty, std::max(fThread->priority - _GetMinimalPriority(), int32(0))); fAdditionalPenalty = currentThreadData->fAdditionalPenalty; @@ -148,21 +140,8 @@ ThreadData::Dump() const additionalPenalty, fAdditionalPenalty); kprintf("\teffective_priority:\t%" B_PRId32 "\n", GetEffectivePriority()); - kprintf("\treceived_penalty:\t%s\n", fReceivedPenalty ? "true" : "false"); - kprintf("\thas_slept:\t\t%s\n", fHasSlept ? "true" : "false"); - - bigtime_t quantum = _GetBaseQuantum(); - if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { - int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount(); - threadCount = max_c(threadCount, 1); - - quantum - = std::min(gCurrentMode->maximum_latency / threadCount, quantum); - quantum = std::max(quantum, gCurrentMode->minimal_quantum); - } - kprintf("\ttime_left:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n", - fTimeLeft, quantum); - + kprintf("\ttime_used:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n", + fTimeUsed, ComputeQuantum()); kprintf("\tstolen_time:\t\t%" B_PRId64 " us\n", fStolenTime); kprintf("\tquantum_start:\t\t%" B_PRId64 " us\n", fQuantumStart); kprintf("\tneeded_load:\t\t%" B_PRId32 "%%\n", fNeededLoad / 10); @@ -206,22 +185,14 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) bigtime_t -ThreadData::ComputeQuantum() +ThreadData::ComputeQuantum() const { SCHEDULER_ENTER_FUNCTION(); - bigtime_t quantum; - if (fTimeLeft != 0) - quantum = fTimeLeft; - else - quantum = _GetBaseQuantum(); - + bigtime_t quantum = _GetBaseQuantum(); if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) return quantum; - quantum += fStolenTime; - fStolenTime = 0; - int32 threadCount = fCore->ThreadCount() / fCore->CPUCount(); if (threadCount >= 1) { quantum @@ -229,7 +200,6 @@ ThreadData::ComputeQuantum() quantum = std::max(quantum, gCurrentMode->minimal_quantum); } - fTimeLeft = quantum; return quantum; } @@ -266,14 +236,7 @@ inline int32 ThreadData::_GetPenalty() const { SCHEDULER_ENTER_FUNCTION(); - - int32 penalty = fPriorityPenalty; - - const int kMinimalPriority = _GetMinimalPriority(); - if (kMinimalPriority > 0) - penalty += fAdditionalPenalty % kMinimalPriority; - - return penalty; + return fPriorityPenalty; } @@ -304,6 +267,8 @@ ThreadData::_ComputeEffectivePriority() const else { fEffectivePriority = fThread->priority; fEffectivePriority -= _GetPenalty(); + if (fEffectivePriority > 0) + fEffectivePriority -= fAdditionalPenalty % fEffectivePriority; ASSERT(fEffectivePriority < B_FIRST_REAL_TIME_PRIORITY); ASSERT(fEffectivePriority >= B_LOWEST_ACTIVE_PRIORITY); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index f3f7020781..af5cd25791 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -56,6 +56,11 @@ public: { fLastInterruptTime = interruptTime; } inline void SetStolenInterruptTime(bigtime_t interruptTime); + bigtime_t ComputeQuantum() const; + inline bigtime_t GetQuantumLeft(); + inline void StartQuantum(); + inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); + inline void Continues(); inline void GoesAway(); inline void Dies(); @@ -69,10 +74,6 @@ public: inline void UpdateActivity(bigtime_t active); - inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); - bigtime_t ComputeQuantum(); - inline void StartQuantum(); - inline bool IsEnqueued() const { return fEnqueued; } inline void SetDequeued() { fEnqueued = false; } @@ -85,7 +86,7 @@ public: static void ComputeQuantumLengths(); private: - inline void _IncreasePenalty(bool strong); + inline void _IncreasePenalty(); inline int32 _GetPenalty() const; void _ComputeNeededLoad(); @@ -112,12 +113,10 @@ private: int32 fPriorityPenalty; int32 fAdditionalPenalty; - bool fReceivedPenalty; - bool fHasSlept; mutable int32 fEffectivePriority; - bigtime_t fTimeLeft; + bigtime_t fTimeUsed; bigtime_t fMeasureAvailableActiveTime; bigtime_t fMeasureAvailableTime; @@ -178,7 +177,7 @@ ThreadData::GetEffectivePriority() const inline void -ThreadData::_IncreasePenalty(bool strong) +ThreadData::_IncreasePenalty() { SCHEDULER_ENTER_FUNCTION(); @@ -189,18 +188,10 @@ ThreadData::_IncreasePenalty(bool strong) TRACE("increasing thread %ld penalty\n", fThread->id); - fReceivedPenalty = true; - int32 oldPenalty = fPriorityPenalty; - if (strong) - fPriorityPenalty++; - - ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - + int32 oldPenalty = fPriorityPenalty++; const int kMinimalPriority = _GetMinimalPriority(); - if (!strong || fThread->priority - oldPenalty <= kMinimalPriority) { + if (fThread->priority - oldPenalty <= kMinimalPriority) fPriorityPenalty = oldPenalty; - fAdditionalPenalty++; - } _ComputeEffectivePriority(); } @@ -210,7 +201,7 @@ inline bool ThreadData::IsCPUBound() const { SCHEDULER_ENTER_FUNCTION(); - return fAdditionalPenalty != 0 && fPriorityPenalty != 0; + return GetThread()->priority - fPriorityPenalty == _GetMinimalPriority(); } @@ -220,8 +211,6 @@ ThreadData::CancelPenalty() SCHEDULER_ENTER_FUNCTION(); int32 oldPenalty = fPriorityPenalty; - - fAdditionalPenalty = 0; fPriorityPenalty = 0; if (oldPenalty != 0) { @@ -239,8 +228,7 @@ ThreadData::ShouldCancelPenalty() const if (fCore == NULL) return false; - if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY - && !IsCPUBound()) { + if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY && !IsCPUBound()) { if (fCore->StarvationCounter() != fWentSleepCount) return true; } @@ -259,6 +247,61 @@ ThreadData::SetStolenInterruptTime(bigtime_t interruptTime) } +inline bigtime_t +ThreadData::GetQuantumLeft() +{ + SCHEDULER_ENTER_FUNCTION(); + + bigtime_t stolenTime = std::min(fStolenTime, gCurrentMode->minimal_quantum); + ASSERT(stolenTime >= 0); + fStolenTime -= stolenTime; + + bigtime_t quantum = ComputeQuantum() - fTimeUsed; + quantum += stolenTime; + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + + return quantum; +} + + +inline void +ThreadData::StartQuantum() +{ + SCHEDULER_ENTER_FUNCTION(); + fQuantumStart = system_time(); +} + + +inline bool +ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) +{ + SCHEDULER_ENTER_FUNCTION(); + + bigtime_t timeUsed = system_time() - fQuantumStart; + ASSERT(timeUsed >= 0); + fTimeUsed += timeUsed; + + bigtime_t timeLeft = ComputeQuantum() - fTimeUsed; + timeLeft = std::max(bigtime_t(0), timeLeft); + + // too little time left, it's better make the next quantum a bit longer + bigtime_t skipTime = gCurrentMode->minimal_quantum / 2; + if (hasYielded || wasPreempted || timeLeft <= skipTime) { + fStolenTime += timeLeft; + timeLeft = 0; + } + + if (timeLeft == 0) { + fAdditionalPenalty++; + _IncreasePenalty(); + fTimeUsed = 0; + return true; + } + + return false; +} + + inline void ThreadData::Continues() { @@ -277,20 +320,17 @@ ThreadData::GoesAway() ASSERT(fReady); - bigtime_t timeUsed = system_time() - fQuantumStart; - ASSERT(timeUsed >= 0); - fTimeLeft -= timeUsed; - - if (!fReceivedPenalty) - _IncreasePenalty(false); - fHasSlept = true; + if (!HasQuantumEnded(false, false)) { + fAdditionalPenalty++; + _ComputeEffectivePriority(); + } fLastInterruptTime = 0; fWentSleep = system_time(); + fWentSleepActive = fCore->GetActiveTime(); fWentSleepCount = fCore->StarvationCounter(); fWentSleepCountIdle = fCore->StarvationCounterIdle(); - fWentSleepActive = fCore->GetActiveTime(); if (gTrackCoreLoad) fCore->UpdateLoad(-fNeededLoad); @@ -415,46 +455,6 @@ ThreadData::UpdateActivity(bigtime_t active) } -inline bool -ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) -{ - SCHEDULER_ENTER_FUNCTION(); - - bigtime_t timeUsed = system_time() - fQuantumStart; - ASSERT(timeUsed >= 0); - fTimeLeft -= timeUsed; - - if (hasYielded) { - fTimeLeft = 0; - return true; - } - - // too little time left, it's better make the next quantum a bit longer - int32 skipTime = gCurrentMode->minimal_quantum / 2; - if (wasPreempted || fTimeLeft <= skipTime) { - fStolenTime += fTimeLeft; - fTimeLeft = 0; - } - - if (fTimeLeft == 0) { - if (!fReceivedPenalty && !fHasSlept) - _IncreasePenalty(true); - fReceivedPenalty = false; - fHasSlept = false; - } - - return fTimeLeft == 0; -} - - -inline void -ThreadData::StartQuantum() -{ - SCHEDULER_ENTER_FUNCTION(); - fQuantumStart = system_time(); -} - - inline void ThreadData::UnassignCore(bool running) { From b7d404c2df546acbe91df76939005bab666dd71b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 16 Jan 2014 22:42:56 +0100 Subject: [PATCH 268/273] scheduler: Add ThreadData::{GetPriority, IsIdle, IsRealTime}() --- src/system/kernel/scheduler/scheduler.cpp | 8 ++--- .../kernel/scheduler/scheduler_thread.cpp | 16 +++++----- .../kernel/scheduler/scheduler_thread.h | 31 ++++++++++++++----- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 557344dc73..2a4dbb12e7 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -388,7 +388,7 @@ reschedule(int32 nextState) case B_THREAD_READY: enqueueOldThread = true; - if (!thread_is_idle_thread(oldThread)) { + if (!oldThreadData->IsIdle()) { oldThreadData->Continues(); if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, oldThread->has_yielded)) { @@ -425,7 +425,7 @@ reschedule(int32 nextState) // select thread with the biggest priority and enqueue back the old thread ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { - if (!thread_is_idle_thread(oldThread)) { + if (!oldThreadData->IsIdle()) { CPURunQueueLocker _(cpu); nextThreadData = cpu->PeekIdleThread(); @@ -446,7 +446,7 @@ reschedule(int32 nextState) } Thread* nextThread = nextThreadData->GetThread(); - ASSERT(!gCPU[thisCPU].disabled || thread_is_idle_thread(nextThread)); + ASSERT(!gCPU[thisCPU].disabled || nextThreadData->IsIdle()); if (nextThread != oldThread) { if (enqueueOldThread) { @@ -483,7 +483,7 @@ reschedule(int32 nextState) cancel_timer(quantumTimer); oldThread->cpu->preempted = false; - if (!thread_is_idle_thread(nextThread)) { + if (!nextThreadData->IsIdle()) { bigtime_t quantum = nextThreadData->GetQuantumLeft(); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index ca6fada228..795fd9f730 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -17,7 +17,7 @@ ThreadData::_InitBase() { fPriorityPenalty = 0; fAdditionalPenalty = 0; - fEffectivePriority = fThread->priority; + fEffectivePriority = GetPriority(); fTimeUsed = 0; fStolenTime = 0; @@ -107,9 +107,9 @@ ThreadData::Init() ThreadData* currentThreadData = currentThread->scheduler_data; fCore = currentThreadData->fCore; - if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { + if (!IsRealTime()) { fPriorityPenalty = std::min(currentThreadData->fPriorityPenalty, - std::max(fThread->priority - _GetMinimalPriority(), int32(0))); + std::max(GetPriority() - _GetMinimalPriority(), int32(0))); fAdditionalPenalty = currentThreadData->fAdditionalPenalty; _ComputeEffectivePriority(); @@ -190,7 +190,7 @@ ThreadData::ComputeQuantum() const SCHEDULER_ENTER_FUNCTION(); bigtime_t quantum = _GetBaseQuantum(); - if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + if (IsRealTime()) return quantum; int32 threadCount = fCore->ThreadCount() / fCore->CPUCount(); @@ -260,12 +260,12 @@ ThreadData::_ComputeEffectivePriority() const { SCHEDULER_ENTER_FUNCTION(); - if (thread_is_idle_thread(fThread)) + if (IsIdle()) fEffectivePriority = B_IDLE_PRIORITY; - else if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) - fEffectivePriority = fThread->priority; + else if (IsRealTime()) + fEffectivePriority = GetPriority(); else { - fEffectivePriority = fThread->priority; + fEffectivePriority = GetPriority(); fEffectivePriority -= _GetPenalty(); if (fEffectivePriority > 0) fEffectivePriority -= fAdditionalPenalty % fEffectivePriority; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index af5cd25791..6e09314968 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -39,6 +39,12 @@ public: void Dump() const; + inline int32 GetPriority() const { return fThread->priority; } + inline Thread* GetThread() const { return fThread; } + + inline bool IsRealTime() const; + inline bool IsIdle() const; + inline bool HasCacheExpired() const; inline bool ShouldRebalance() const; @@ -77,7 +83,6 @@ public: inline bool IsEnqueued() const { return fEnqueued; } inline void SetDequeued() { fEnqueued = false; } - inline Thread* GetThread() const { return fThread; } inline int32 GetLoad() const { return fNeededLoad; } inline CoreEntry* Core() const { return fCore; } @@ -145,11 +150,25 @@ ThreadData::_GetMinimalPriority() const const int32 kMaximalPriority = 25; const int32 kMinimalPriority = B_LOWEST_ACTIVE_PRIORITY; - int32 priority = fThread->priority / kDivisor; + int32 priority = GetPriority() / kDivisor; return std::max(std::min(priority, kMaximalPriority), kMinimalPriority); } +inline bool +ThreadData::IsRealTime() const +{ + return GetPriority() >= B_FIRST_REAL_TIME_PRIORITY; +} + + +inline bool +ThreadData::IsIdle() const +{ + return GetPriority() == B_IDLE_PRIORITY; +} + + inline bool ThreadData::HasCacheExpired() const { @@ -181,16 +200,14 @@ ThreadData::_IncreasePenalty() { SCHEDULER_ENTER_FUNCTION(); - if (fThread->priority < B_LOWEST_ACTIVE_PRIORITY) - return; - if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + if (IsIdle() || IsRealTime()) return; TRACE("increasing thread %ld penalty\n", fThread->id); int32 oldPenalty = fPriorityPenalty++; const int kMinimalPriority = _GetMinimalPriority(); - if (fThread->priority - oldPenalty <= kMinimalPriority) + if (GetPriority() - oldPenalty <= kMinimalPriority) fPriorityPenalty = oldPenalty; _ComputeEffectivePriority(); @@ -201,7 +218,7 @@ inline bool ThreadData::IsCPUBound() const { SCHEDULER_ENTER_FUNCTION(); - return GetThread()->priority - fPriorityPenalty == _GetMinimalPriority(); + return GetPriority() - fPriorityPenalty == _GetMinimalPriority(); } From f978518a52465f8b3cbf8ea1fd27541a5d54af11 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 16 Jan 2014 23:21:10 +0100 Subject: [PATCH 269/273] scheudler: Cache ThreadData::IsCPUBound() result --- src/system/kernel/scheduler/scheduler_thread.cpp | 1 + src/system/kernel/scheduler/scheduler_thread.h | 16 ++++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 795fd9f730..d778b35c43 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -18,6 +18,7 @@ ThreadData::_InitBase() fPriorityPenalty = 0; fAdditionalPenalty = 0; fEffectivePriority = GetPriority(); + fCPUBound = false; fTimeUsed = 0; fStolenTime = 0; diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 6e09314968..bfcaa59e77 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -50,7 +50,7 @@ public: inline int32 GetEffectivePriority() const; - inline bool IsCPUBound() const; + inline bool IsCPUBound() const { return fCPUBound; } inline void CancelPenalty(); inline bool ShouldCancelPenalty() const; @@ -116,6 +116,7 @@ private: Thread* fThread; + bool fCPUBound; int32 fPriorityPenalty; int32 fAdditionalPenalty; @@ -207,21 +208,15 @@ ThreadData::_IncreasePenalty() int32 oldPenalty = fPriorityPenalty++; const int kMinimalPriority = _GetMinimalPriority(); - if (GetPriority() - oldPenalty <= kMinimalPriority) + if (GetPriority() - oldPenalty <= kMinimalPriority) { fPriorityPenalty = oldPenalty; + fCPUBound = true; + } _ComputeEffectivePriority(); } -inline bool -ThreadData::IsCPUBound() const -{ - SCHEDULER_ENTER_FUNCTION(); - return GetPriority() - fPriorityPenalty == _GetMinimalPriority(); -} - - inline void ThreadData::CancelPenalty() { @@ -229,6 +224,7 @@ ThreadData::CancelPenalty() int32 oldPenalty = fPriorityPenalty; fPriorityPenalty = 0; + fCPUBound = false; if (oldPenalty != 0) { TRACE("cancelling thread %ld penalty\n", fThread->id); From 0d27a57cf17e7e60a8fbb095500e00d8c951fc80 Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 16 Jan 2014 23:24:34 +0100 Subject: [PATCH 270/273] scheduler: Fix divide by zero on CPU disable --- src/system/kernel/scheduler/scheduler_thread.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index d778b35c43..29d3ae2975 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -194,7 +194,9 @@ ThreadData::ComputeQuantum() const if (IsRealTime()) return quantum; - int32 threadCount = fCore->ThreadCount() / fCore->CPUCount(); + int32 threadCount = fCore->ThreadCount(); + if (fCore->CPUCount() > 0) + threadCount /= fCore->CPUCount(); if (threadCount >= 1) { quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum); From 4835c216f783497c2a8e1bc8eea073925597148d Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Thu, 16 Jan 2014 23:45:28 +0100 Subject: [PATCH 271/273] scheduler: low_latency: Try to keep all cores busy --- src/system/kernel/scheduler/low_latency.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 20555e582c..911353f2a1 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -101,10 +101,14 @@ should_rebalance(const ThreadData* threadData) // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap.PeekMinimum(); if (other == NULL) other = gCoreHighLoadHeap.PeekMinimum(); + coreLocker.Unlock(); + + if (other->GetLoad() == 0 && coreNewLoad != 0) + return true; + int32 otherNewLoad = other->GetLoad() + threadLoad; return coreNewLoad - otherNewLoad >= kLoadDifference * 2; } From 3dce49af0ecc67f243fdd37fbcfa3321f7d2047b Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 17 Jan 2014 01:43:41 +0100 Subject: [PATCH 272/273] scheduler: Cache quantum length --- .../kernel/scheduler/scheduler_thread.cpp | 36 ++++++++++--------- .../kernel/scheduler/scheduler_thread.h | 3 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index 29d3ae2975..8651c5cacc 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -11,6 +11,9 @@ using namespace Scheduler; static bigtime_t sQuantumLengths[THREAD_MAX_SET_PRIORITY + 1]; +const int32 kMaximumQuantumLengthsCount = 20; +static bigtime_t sMaximumQuantumLengths[kMaximumQuantumLengthsCount]; + void ThreadData::_InitBase() @@ -18,6 +21,7 @@ ThreadData::_InitBase() fPriorityPenalty = 0; fAdditionalPenalty = 0; fEffectivePriority = GetPriority(); + fBaseQuantum = sQuantumLengths[GetEffectivePriority()]; fCPUBound = false; fTimeUsed = 0; @@ -83,15 +87,6 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const } -inline bigtime_t -ThreadData::_GetBaseQuantum() const -{ - SCHEDULER_ENTER_FUNCTION(); - - return sQuantumLengths[GetEffectivePriority()]; -} - - ThreadData::ThreadData(Thread* thread) : fThread(thread) @@ -190,19 +185,16 @@ ThreadData::ComputeQuantum() const { SCHEDULER_ENTER_FUNCTION(); - bigtime_t quantum = _GetBaseQuantum(); if (IsRealTime()) - return quantum; + return fBaseQuantum; int32 threadCount = fCore->ThreadCount(); if (fCore->CPUCount() > 0) threadCount /= fCore->CPUCount(); - if (threadCount >= 1) { - quantum - = std::min(gCurrentMode->maximum_latency / threadCount, quantum); - quantum = std::max(quantum, gCurrentMode->minimal_quantum); - } + bigtime_t quantum = fBaseQuantum; + if (threadCount < kMaximumQuantumLengthsCount) + quantum = std::min(sMaximumQuantumLengths[threadCount], quantum); return quantum; } @@ -232,6 +224,16 @@ ThreadData::ComputeQuantumLengths() sQuantumLengths[priority] = _ScaleQuantum(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, B_IDLE_PRIORITY, priority); } + + for (int32 threadCount = 0; threadCount < kMaximumQuantumLengthsCount; + threadCount++) { + + bigtime_t quantum = gCurrentMode->maximum_latency; + if (threadCount != 0) + quantum /= threadCount; + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + sMaximumQuantumLengths[threadCount] = quantum; + } } @@ -276,6 +278,8 @@ ThreadData::_ComputeEffectivePriority() const ASSERT(fEffectivePriority < B_FIRST_REAL_TIME_PRIORITY); ASSERT(fEffectivePriority >= B_LOWEST_ACTIVE_PRIORITY); } + + fBaseQuantum = sQuantumLengths[GetEffectivePriority()]; } diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index bfcaa59e77..980d6c2fff 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -29,8 +29,6 @@ private: inline CPUEntry* _ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const; - inline bigtime_t _GetBaseQuantum() const; - public: ThreadData(Thread* thread); @@ -121,6 +119,7 @@ private: int32 fAdditionalPenalty; mutable int32 fEffectivePriority; + mutable bigtime_t fBaseQuantum; bigtime_t fTimeUsed; From 0269dd284e334fef121a93547cf2a81e065109bb Mon Sep 17 00:00:00 2001 From: Pawel Dziepak Date: Fri, 17 Jan 2014 01:44:46 +0100 Subject: [PATCH 273/273] Revert "scheduler: SCHEDULER_EXIT_FUNCTION() is not needed" This reverts commit 667617ad043a4587d8d366d5192d9ad291cfa37a. Scheduler profiler uses CPU local data to store function information, hence arch_thread_context_switch() usually is not a problem. However, when we switch to a new thread we end up scheduler_new_thread_entry() instead of scheduler_reschedule() what may corrupt data collected by the profiler. --- src/system/kernel/scheduler/scheduler.cpp | 2 ++ src/system/kernel/scheduler/scheduler_profiler.h | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 2a4dbb12e7..6437cb0d32 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -495,6 +495,8 @@ reschedule(int32 nextState) modeLocker.Unlock(); + SCHEDULER_EXIT_FUNCTION(); + if (nextThread != oldThread) switch_thread(oldThread, nextThread); } diff --git a/src/system/kernel/scheduler/scheduler_profiler.h b/src/system/kernel/scheduler/scheduler_profiler.h index e6c95b5703..d2c1bfca3d 100644 --- a/src/system/kernel/scheduler/scheduler_profiler.h +++ b/src/system/kernel/scheduler/scheduler_profiler.h @@ -16,6 +16,9 @@ #define SCHEDULER_ENTER_FUNCTION() \ Scheduler::Profiling::Function schedulerProfiler(__PRETTY_FUNCTION__) +#define SCHEDULER_EXIT_FUNCTION() \ + schedulerProfiler.Exit() + namespace Scheduler { @@ -124,6 +127,7 @@ Function::Exit() #else // SCHEDULER_PROFILING #define SCHEDULER_ENTER_FUNCTION() (void)0 +#define SCHEDULER_EXIT_FUNCTION() (void)0 #endif // !SCHEDULER_PROFILING