diff --git a/headers/private/kernel/thread_types.h b/headers/private/kernel/thread_types.h index 2a7051bc3b..eb7f1ad5e1 100644 --- a/headers/private/kernel/thread_types.h +++ b/headers/private/kernel/thread_types.h @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -58,12 +57,15 @@ struct cpu_ent; struct image; // defined in image.c struct io_context; struct realtime_sem_context; // defined in realtime_sem.cpp -struct scheduler_thread_data; struct select_info; struct user_thread; // defined in libroot/user_thread.h struct VMAddressSpace; struct xsi_sem_context; // defined in xsi_semaphore.cpp +namespace Scheduler { + struct ThreadData; +} + namespace BKernel { struct Team; struct Thread; @@ -412,8 +414,7 @@ private: }; -struct Thread : TeamThreadIteratorEntry, KernelReferenceable, - RunQueueLinkImpl { +struct Thread : TeamThreadIteratorEntry, KernelReferenceable { int32 flags; // summary of events relevant in interrupt // handlers (signals pending, user debugging // enabled, etc.) @@ -444,7 +445,7 @@ struct Thread : TeamThreadIteratorEntry, KernelReferenceable, bool in_kernel; // protected by time_lock, only written by // this thread bool has_yielded; // protected by scheduler lock - struct scheduler_thread_data* scheduler_data; // protected by scheduler lock + Scheduler::ThreadData* scheduler_data; // protected by scheduler lock struct user_thread* user_thread; // write-protected by fLock, only // modified by the thread itself and diff --git a/headers/private/kernel/util/atomic.h b/headers/private/kernel/util/atomic.h index 14725492c8..40b057c608 100644 --- a/headers/private/kernel/util/atomic.h +++ b/headers/private/kernel/util/atomic.h @@ -10,6 +10,8 @@ #include +#include + #ifdef __cplusplus diff --git a/src/system/kernel/Jamfile b/src/system/kernel/Jamfile index 4173486dff..ab80c5d31a 100644 --- a/src/system/kernel/Jamfile +++ b/src/system/kernel/Jamfile @@ -65,6 +65,8 @@ KernelMergeObject kernel_core.o : low_latency.cpp power_saving.cpp scheduler.cpp + scheduler_cpu.cpp + scheduler_thread.cpp scheduler_tracing.cpp scheduling_analysis.cpp diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 6adfe4e6fb..a272f53214 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -7,7 +7,9 @@ #include #include "scheduler_common.h" +#include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_thread.h" using namespace Scheduler; @@ -29,16 +31,12 @@ set_cpu_enabled(int32 /* cpu */, bool /* enabled */) static bool -has_cache_expired(Thread* thread) +has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; - return atomic_get64(&coreEntry->fActiveTime) - - schedulerThreadData->went_sleep_active > kCacheExpire; + return atomic_get64(&threadData->GetCore()->fActiveTime) + - threadData->fWentSleepActive > kCacheExpire; } @@ -58,51 +56,45 @@ get_most_idle_package(void) } -static int32 -choose_core(Thread* thread) +static CoreEntry* +choose_core(const ThreadData* /* threadData */) { - CoreEntry* entry = NULL; - ReadSpinLocker locker(gIdlePackageLock); // wake new package - PackageEntry* package = gIdlePackageList->Last(); + PackageEntry* package = gIdlePackageList.Last(); if (package == NULL) { // wake new core package = get_most_idle_package(); } locker.Unlock(); + CoreEntry* core = NULL; if (package != NULL) { ReadSpinLocker _(package->fCoreLock); - entry = package->fIdleCores.Last(); + core = package->fIdleCores.Last(); } - if (entry == NULL) { + if (core == NULL) { ReadSpinLocker coreLocker(gCoreHeapsLock); // no idle cores, use least occupied core - entry = gCoreLoadHeap->PeekMinimum(); - if (entry == NULL) - entry = gCoreHighLoadHeap->PeekMinimum(); + core = gCoreLoadHeap.PeekMinimum(); + if (core == NULL) + core = gCoreHighLoadHeap.PeekMinimum(); } - ASSERT(entry != NULL); - return entry->fCoreID; + ASSERT(core != NULL); + return core; } static bool -should_rebalance(Thread* thread) +should_rebalance(const ThreadData* threadData) { - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - CoreEntry* coreEntry = &gCoreEntries[schedulerThreadData->previous_core]; - - int32 coreLoad = get_core_load(coreEntry); + int32 coreLoad = threadData->GetCore()->GetLoad(); // If the thread produces more than 50% of the load, leave it here. In // such situation it is better to move other threads away. - if (schedulerThreadData->load >= coreLoad / 2) + if (threadData->GetLoad() >= coreLoad / 2) return false; // If there is high load on this core but this thread does not contribute @@ -110,22 +102,20 @@ should_rebalance(Thread* thread) if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); - if (other != NULL && coreLoad - get_core_load(other) - >= kLoadDifference) { + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); + if (other != NULL && coreLoad - other->GetLoad() >= kLoadDifference) return true; - } } // No cpu bound threads - the situation is quite good. Make sure it // won't get much worse... ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); + other = gCoreHighLoadHeap.PeekMinimum(); ASSERT(other != NULL); - return coreLoad - get_core_load(other) >= kLoadDifference * 2; + return coreLoad - other->GetLoad() >= kLoadDifference * 2; } @@ -155,26 +145,22 @@ rebalance_irqs(bool idle) return; ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); + other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; + int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; cpuLocker.Unlock(); - ASSERT(other != NULL); - int32 thisCore = gCPUToCore[smp_get_current_cpu()]; - if (other->fCoreID == thisCore) + CoreEntry* core = CoreEntry::GetCore(cpu->cpu_num); + if (other == core) return; - - if (get_core_load(other) + kLoadDifference - >= get_core_load(&gCoreEntries[thisCore])) { + if (other->GetLoad() + kLoadDifference >= core->GetLoad()) return; - } assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 9613a356ce..c1eeb61609 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -4,10 +4,13 @@ */ +#include #include #include "scheduler_common.h" +#include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_thread.h" using namespace Scheduler; @@ -15,13 +18,13 @@ using namespace Scheduler; const bigtime_t kCacheExpire = 100000; -static int32 sSmallTaskCore; +static CoreEntry* sSmallTaskCore; static void switch_to_mode(void) { - sSmallTaskCore = -1; + sSmallTaskCore = NULL; } @@ -29,34 +32,32 @@ static void set_cpu_enabled(int32 cpu, bool enabled) { if (!enabled) - sSmallTaskCore = -1; + sSmallTaskCore = NULL; } static bool -has_cache_expired(Thread* thread) +has_cache_expired(const ThreadData* threadData) { ASSERT(!gSingleCore); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); - - return system_time() - schedulerThreadData->went_sleep > kCacheExpire; + return system_time() - threadData->fWentSleep > kCacheExpire; } -static int32 +static CoreEntry* choose_small_task_core(void) { ReadSpinLocker locker(gCoreHeapsLock); - CoreEntry* candidate = gCoreLoadHeap->PeekMaximum(); + CoreEntry* core = gCoreLoadHeap.PeekMaximum(); locker.Unlock(); - if (candidate == NULL) + + if (core == NULL) return sSmallTaskCore; - int32 core = candidate->fCoreID; - int32 smallTaskCore = atomic_test_and_set(&sSmallTaskCore, core, -1); - if (smallTaskCore == -1) + CoreEntry* smallTaskCore + = atomic_pointer_test_and_set(&sSmallTaskCore, core, (CoreEntry*)NULL); + if (smallTaskCore == NULL) return core; return smallTaskCore; } @@ -65,121 +66,118 @@ choose_small_task_core(void) static CoreEntry* choose_idle_core(void) { - PackageEntry* current = NULL; + PackageEntry* package = NULL; + for (int32 i = 0; i < gPackageCount; i++) { - if (gPackageEntries[i].fIdleCoreCount != 0 && (current == NULL - || gPackageEntries[i].fIdleCoreCount - < current->fIdleCoreCount)) { - current = &gPackageEntries[i]; + PackageEntry* current = &gPackageEntries[i]; + if (current->fIdleCoreCount != 0 && (package == NULL + || current->fIdleCoreCount < package->fIdleCoreCount)) { + package = current; } } - if (current == NULL) { + if (package == NULL) { ReadSpinLocker _(gIdlePackageLock); - current = gIdlePackageList->Last(); + package = gIdlePackageList.Last(); } - if (current != NULL) { - ReadSpinLocker _(current->fCoreLock); - return current->fIdleCores.Last(); + if (package != NULL) { + ReadSpinLocker _(package->fCoreLock); + return package->fIdleCores.Last(); } return NULL; } -static int32 -choose_core(Thread* thread) +static CoreEntry* +choose_core(const ThreadData* threadData) { - CoreEntry* entry; + CoreEntry* core = NULL; - int32 core = -1; // try to pack all threads on one core core = choose_small_task_core(); - if (core != -1 - && get_core_load(&gCoreEntries[core]) + thread->scheduler_data->load - < kHighLoad) { - entry = &gCoreEntries[core]; - } else { + if (core == NULL || core->GetLoad() + threadData->GetLoad() >= kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); + // run immediately on already woken core - entry = gCoreLoadHeap->PeekMinimum(); - if (entry == NULL) { + core = gCoreLoadHeap.PeekMinimum(); + if (core == NULL) { coreLocker.Unlock(); - entry = choose_idle_core(); + core = choose_idle_core(); - if (entry == NULL) { + if (core == NULL) { coreLocker.Lock(); - entry = gCoreHighLoadHeap->PeekMinimum(); + core = gCoreHighLoadHeap.PeekMinimum(); } } } - ASSERT(entry != NULL); - return entry->fCoreID; + ASSERT(core != NULL); + return core; } static bool -should_rebalance(Thread* thread) +should_rebalance(const ThreadData* threadData) { ASSERT(!gSingleCore); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - ASSERT(schedulerThreadData->previous_core >= 0); + CoreEntry* core = threadData->GetCore(); - int32 core = schedulerThreadData->previous_core; - CoreEntry* coreEntry = &gCoreEntries[core]; - - int32 coreLoad = get_core_load(coreEntry); + int32 coreLoad = core->GetLoad(); if (coreLoad > kHighLoad) { ReadSpinLocker coreLocker(gCoreHeapsLock); if (sSmallTaskCore == core) { - sSmallTaskCore = -1; + sSmallTaskCore = NULL; choose_small_task_core(); - if (schedulerThreadData->load > coreLoad / 3) + + if (threadData->GetLoad() > coreLoad / 3) return false; return coreLoad > kVeryHighLoad; } - if (schedulerThreadData->load >= coreLoad / 2) + if (threadData->GetLoad() >= coreLoad / 2) return false; - CoreEntry* other = gCoreLoadHeap->PeekMaximum(); + CoreEntry* other = gCoreLoadHeap.PeekMaximum(); if (other == NULL) - other = gCoreHighLoadHeap->PeekMinimum(); + other = gCoreHighLoadHeap.PeekMinimum(); ASSERT(other != NULL); - return coreLoad - get_core_load(other) >= kLoadDifference / 2; + return coreLoad - other->GetLoad() >= kLoadDifference / 2; } if (coreLoad >= kMediumLoad) return false; - int32 smallTaskCore = choose_small_task_core(); - if (smallTaskCore == -1) + CoreEntry* smallTaskCore = choose_small_task_core(); + if (smallTaskCore == NULL) return false; return smallTaskCore != core - && get_core_load(&gCoreEntries[smallTaskCore]) - + thread->scheduler_data->load < kHighLoad; + && smallTaskCore->GetLoad() +threadData->GetLoad() < kHighLoad; } static inline void pack_irqs(void) { + CoreEntry* smallTaskCore = atomic_pointer_get(&sSmallTaskCore); + if (smallTaskCore == NULL) + return; + cpu_ent* cpu = get_cpu_struct(); - int32 core = gCPUToCore[cpu->cpu_num]; + if (smallTaskCore == CoreEntry::GetCore(cpu->cpu_num)) + return; SpinLocker locker(cpu->irqs_lock); - while (sSmallTaskCore != core && list_get_first_item(&cpu->irqs) != NULL) { + while (list_get_first_item(&cpu->irqs) != NULL) { irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); ReadSpinLocker coreLocker(gCoreHeapsLock); - int32 newCPU - = gCPUPriorityHeaps[sSmallTaskCore].PeekMinimum()->fCPUNumber; + int32 newCPU = smallTaskCore->fCPUHeap.PeekMinimum()->fCPUNumber; coreLocker.Unlock(); if (newCPU != cpu->cpu_num) @@ -193,12 +191,12 @@ pack_irqs(void) static void rebalance_irqs(bool idle) { - if (idle && sSmallTaskCore != -1) { + if (idle && sSmallTaskCore != NULL) { pack_irqs(); return; } - if (idle || sSmallTaskCore != -1) + if (idle || sSmallTaskCore != NULL) return; cpu_ent* cpu = get_cpu_struct(); @@ -219,22 +217,19 @@ rebalance_irqs(bool idle) return; ReadSpinLocker coreLocker(gCoreHeapsLock); - CoreEntry* other = gCoreLoadHeap->PeekMinimum(); + CoreEntry* other = gCoreLoadHeap.PeekMinimum(); coreLocker.Unlock(); if (other == NULL) return; SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = gCPUPriorityHeaps[other->fCoreID].PeekMinimum()->fCPUNumber; + int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; cpuLocker.Unlock(); - int32 thisCore = gCPUToCore[smp_get_current_cpu()]; - if (other->fCoreID == thisCore) + CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); + if (other == core) return; - - if (get_core_load(other) + kLoadDifference - >= get_core_load(&gCoreEntries[thisCore])) { + if (other->GetLoad() + kLoadDifference >= core->GetLoad()) return; - } assign_io_interrupt_to_cpu(chosen->irq, newCPU); } diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 6f71982e58..18c878470e 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -29,13 +29,90 @@ #include #include -#include - #include "scheduler_common.h" +#include "scheduler_cpu.h" #include "scheduler_modes.h" +#include "scheduler_thread.h" #include "scheduler_tracing.h" +namespace Scheduler { + + +class SchedulerModeLocker : public ReadSpinLocker { +public: + SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true) + : + ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, + alreadyLocked, lockIfNotLocked) + { + } +}; + +class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { +public: + InterruptsSchedulerModeLocker(bool alreadyLocked = false, + bool lockIfNotLocked = true) + : + InterruptsReadSpinLocker( + gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, + alreadyLocked, lockIfNotLocked) + { + } +}; + +class BigSchedulerLocking { +public: + bool Lock(int* lockable) + { + *lockable = disable_interrupts(); + for (int32 i = 0; i < smp_get_num_cpus(); i++) + acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + return true; + } + + void Unlock(int* lockable) + { + for (int32 i = 0; i < smp_get_num_cpus(); i++) + release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); + restore_interrupts(*lockable); + } +}; + +class InterruptsBigSchedulerLocker : + public AutoLocker { +public: + InterruptsBigSchedulerLocker() + : + AutoLocker(&fState, false, true) + { + } + +private: + int fState; +}; + +scheduler_mode gCurrentModeID; +scheduler_mode_operations* gCurrentMode; + +bool gSingleCore; + +CPUEntry* gCPUEntries; + +CoreEntry* gCoreEntries; +CoreLoadHeap gCoreLoadHeap; +CoreLoadHeap gCoreHighLoadHeap; +rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; +int32 gCoreCount; + +PackageEntry* gPackageEntries; +IdlePackageList gIdlePackageList; +rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; +int32 gPackageCount; + + +} // namespace Scheduler + using namespace Scheduler; @@ -44,581 +121,22 @@ static bool sSchedulerEnabled; SchedulerListenerList gSchedulerListeners; spinlock gSchedulerListenersLock = B_SPINLOCK_INITIALIZER; -static scheduler_mode sCurrentModeID; -static scheduler_mode_operations* sCurrentMode; static scheduler_mode_operations* sSchedulerModes[] = { &gSchedulerLowLatencyMode, &gSchedulerPowerSavingMode, }; -namespace Scheduler { - -bool gSingleCore; - -CPUEntry* gCPUEntries; -CPUHeap* gCPUPriorityHeaps; - -CoreEntry* gCoreEntries; -CoreLoadHeap* gCoreLoadHeap; -CoreLoadHeap* gCoreHighLoadHeap; -rw_spinlock gCoreHeapsLock = B_RW_SPINLOCK_INITIALIZER; -int32 gCoreCount; - -PackageEntry* gPackageEntries; -IdlePackageList* gIdlePackageList; -rw_spinlock gIdlePackageLock = B_RW_SPINLOCK_INITIALIZER; -int32 gPackageCount; - -ThreadRunQueue* gRunQueues; -ThreadRunQueue* gPinnedRunQueues; - -int32* gCPUToCore; -int32* gCPUToPackage; - -class SchedulerModeLocker : public ReadSpinLocker { -public: - inline SchedulerModeLocker(bool alreadyLocked = false, - bool lockIfNotLocked = true) - : - ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) - { - } -}; - -class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker { -public: - inline InterruptsSchedulerModeLocker(bool alreadyLocked = false, - bool lockIfNotLocked = true) - : - InterruptsReadSpinLocker( - gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock, - alreadyLocked, lockIfNotLocked) - { - } -}; - -} // namespace Scheduler - -static CPUHeap* sDebugCPUHeap; -static CoreLoadHeap* sDebugCoreHeap; - - -CPUEntry::CPUEntry() - : - fPriority(B_IDLE_PRIORITY), - fMeasureActiveTime(0), - fMeasureTime(0), - fLoad(0) -{ - B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); -} - - -CoreEntry::CoreEntry() - : - fCPUCount(0), - fStarvationCounter(0), - fThreadCount(0), - fActiveTime(0), - fLoad(0), - fHighLoad(false) -{ - B_INITIALIZE_SPINLOCK(&fCPULock); - B_INITIALIZE_SPINLOCK(&fQueueLock); -} - - -PackageEntry::PackageEntry() - : - fIdleCoreCount(0), - fCoreCount(0) -{ - B_INITIALIZE_RW_SPINLOCK(&fCoreLock); -} - - -scheduler_thread_data::scheduler_thread_data() -{ - Init(); -} - - -void -scheduler_thread_data::Init() -{ - priority_penalty = 0; - additional_penalty = 0; - - time_left = 0; - stolen_time = 0; - - measure_active_time = 0; - measure_time = 0; - load = 0; - - went_sleep = 0; - went_sleep_active = 0; - went_sleep_count = -1; - - previous_core = -1; - enqueued = false; -} - - -static void -dump_queue(ThreadRunQueue::ConstIterator& iterator) -{ - if (!iterator.HasNext()) - kprintf("Run queue is empty.\n"); - else { - kprintf("thread id priority penalty name\n"); - while (iterator.HasNext()) { - Thread* thread = iterator.Next(); - kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", - thread, thread->id, thread->priority, - get_thread_penalty(thread), thread->name); - } - } -} - - -static int -dump_run_queue(int argc, char **argv) -{ - int32 cpuCount = smp_get_num_cpus(); - int32 coreCount = gCoreCount; - - ThreadRunQueue::ConstIterator iterator; - for (int32 i = 0; i < coreCount; i++) { - kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); - iterator = gRunQueues[i].GetConstIterator(); - dump_queue(iterator); - } - - for (int32 i = 0; i < cpuCount; i++) { - iterator = gPinnedRunQueues[i].GetConstIterator(); - - if (iterator.HasNext() && !thread_is_idle_thread(iterator.Next())) { - kprintf("\nCPU %" B_PRId32 " run queue:\n", i); - dump_queue(iterator); - } - } - - return 0; -} - - -static void -dump_cpu_load_heap(CPUHeap* heap) -{ - kprintf("cpu priority load\n"); - CPUEntry* entry = heap->PeekMinimum(); - while (entry) { - int32 cpu = entry->fCPUNumber; - int32 key = CPUHeap::GetKey(entry); - kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, - gCPUEntries[cpu].fLoad / 10); - - heap->RemoveMinimum(); - sDebugCPUHeap->Insert(entry, key); - - entry = heap->PeekMinimum(); - } - - entry = sDebugCPUHeap->PeekMinimum(); - while (entry) { - int32 key = CPUHeap::GetKey(entry); - sDebugCPUHeap->RemoveMinimum(); - heap->Insert(entry, key); - entry = sDebugCPUHeap->PeekMinimum(); - } -} - - -static void -dump_core_load_heap(CoreLoadHeap* heap) -{ - CoreEntry* entry = heap->PeekMinimum(); - while (entry) { - int32 key = CoreLoadHeap::GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, - get_core_load(entry) / 10); - - heap->RemoveMinimum(); - sDebugCoreHeap->Insert(entry, key); - - entry = heap->PeekMinimum(); - } - - entry = sDebugCoreHeap->PeekMinimum(); - while (entry) { - int32 key = CoreLoadHeap::GetKey(entry); - sDebugCoreHeap->RemoveMinimum(); - heap->Insert(entry, key); - entry = sDebugCoreHeap->PeekMinimum(); - } -} - - -static int -dump_cpu_heap(int argc, char** argv) -{ - kprintf("core load\n"); - dump_core_load_heap(gCoreLoadHeap); - kprintf("\n"); - dump_core_load_heap(gCoreHighLoadHeap); - - for (int32 i = 0; i < gCoreCount; i++) { - if (gCoreEntries[i].fCPUCount < 2) - continue; - - kprintf("\nCore %" B_PRId32 " heap:\n", i); - dump_cpu_load_heap(&gCPUPriorityHeaps[i]); - } - - return 0; -} - - -static int -dump_idle_cores(int argc, char** argv) -{ - kprintf("Idle packages:\n"); - IdlePackageList::ReverseIterator idleIterator - = gIdlePackageList->GetReverseIterator(); - - if (idleIterator.HasNext()) { - kprintf("package cores\n"); - - while (idleIterator.HasNext()) { - PackageEntry* entry = idleIterator.Next(); - kprintf("%-7" B_PRId32 " ", entry->fPackageID); - - DoublyLinkedList::ReverseIterator iterator - = entry->fIdleCores.GetReverseIterator(); - if (iterator.HasNext()) { - while (iterator.HasNext()) { - CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, - iterator.HasNext() ? ", " : ""); - } - } else - kprintf("-"); - kprintf("\n"); - } - } else - kprintf("No idle packages.\n"); - - return 0; -} - - -static inline bool -has_cache_expired(Thread* thread) -{ - return sCurrentMode->has_cache_expired(thread); -} +// Since CPU IDs used internally by the kernel bear no relation to the actual +// CPU topology the following arrays are used to efficiently get the core +// and the package that CPU in question belongs to. +static int32* sCPUToCore; +static int32* sCPUToPackage; void scheduler_dump_thread_data(Thread* thread) { - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", - schedulerThreadData->priority_penalty); - - int32 additionalPenalty = 0; - const int kMinimalPriority = get_minimal_priority(thread); - if (kMinimalPriority > 0) { - additionalPenalty - = schedulerThreadData->additional_penalty % kMinimalPriority; - } - kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", - additionalPenalty, schedulerThreadData->additional_penalty); - kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", - schedulerThreadData->stolen_time); - kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", schedulerThreadData->load / 10); - kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", - schedulerThreadData->went_sleep); - kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", - schedulerThreadData->went_sleep_active); - kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", - schedulerThreadData->went_sleep_count); - kprintf("\tprevious_core:\t\t%" B_PRId32 "\n", - schedulerThreadData->previous_core); - if (schedulerThreadData->previous_core > 0 - && has_cache_expired(thread)) { - kprintf("\tcache affinity has expired\n"); - } -} - - -static void -update_load_heaps(int32 core) -{ - ASSERT(!gSingleCore); - - CoreEntry* entry = &gCoreEntries[core]; - - if (entry->fCPUCount == 0) { - entry->fLoad = 0; - return; - } - - WriteSpinLocker coreLocker(gCoreHeapsLock); - - int32 newKey = get_core_load(entry); - int32 oldKey = CoreLoadHeap::GetKey(entry); - - ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); - ASSERT(newKey >= 0 && newKey <= kMaxLoad); - - if (oldKey == newKey) - return; - - if (newKey > kHighLoad) { - if (!entry->fHighLoad) { - gCoreLoadHeap->ModifyKey(entry, -1); - ASSERT(gCoreLoadHeap->PeekMinimum() == entry); - gCoreLoadHeap->RemoveMinimum(); - - gCoreHighLoadHeap->Insert(entry, newKey); - - entry->fHighLoad = true; - } else - gCoreHighLoadHeap->ModifyKey(entry, newKey); - } else if (newKey < kMediumLoad) { - if (entry->fHighLoad) { - gCoreHighLoadHeap->ModifyKey(entry, -1); - ASSERT(gCoreHighLoadHeap->PeekMinimum() == entry); - gCoreHighLoadHeap->RemoveMinimum(); - - gCoreLoadHeap->Insert(entry, newKey); - - entry->fHighLoad = false; - } else - gCoreLoadHeap->ModifyKey(entry, newKey); - } else { - if (entry->fHighLoad) - gCoreHighLoadHeap->ModifyKey(entry, newKey); - else - gCoreLoadHeap->ModifyKey(entry, newKey); - } -} - - -static inline void -increase_penalty(Thread* thread) -{ - if (thread->priority < B_LOWEST_ACTIVE_PRIORITY) - return; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return; - - TRACE("increasing thread %ld penalty\n", thread->id); - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - int32 oldPenalty = schedulerThreadData->priority_penalty++; - - ASSERT(thread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); - - const int kMinimalPriority = get_minimal_priority(thread); - if (thread->priority - oldPenalty <= kMinimalPriority) { - schedulerThreadData->priority_penalty = oldPenalty; - schedulerThreadData->additional_penalty++; - } -} - - -static inline void -cancel_penalty(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (schedulerThreadData->priority_penalty != 0) - TRACE("cancelling thread %ld penalty\n", thread->id); - - schedulerThreadData->additional_penalty = 0; - schedulerThreadData->priority_penalty = 0; -} - - -static inline void -update_cpu_priority(int32 cpu, int32 priority) -{ - int32 core = gCPUToCore[cpu]; - int32 corePriority = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); - gCPUPriorityHeaps[core].ModifyKey(&gCPUEntries[cpu], priority); - - if (gSingleCore) - return; - - int32 maxPriority - = CPUHeap::GetKey(gCPUPriorityHeaps[core].PeekMaximum()); - - if (corePriority == maxPriority) - return; - - int32 package = gCPUToPackage[cpu]; - PackageEntry* packageEntry = &gPackageEntries[package]; - if (maxPriority == B_IDLE_PRIORITY) { - WriteSpinLocker _(packageEntry->fCoreLock); - - // core goes idle - ASSERT(packageEntry->fIdleCoreCount >= 0); - ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount++; - packageEntry->fIdleCores.Add(&gCoreEntries[core]); - - if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { - // package goes idle - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList->Add(packageEntry); - } - } else if (corePriority == B_IDLE_PRIORITY) { - WriteSpinLocker _(packageEntry->fCoreLock); - - // core wakes up - ASSERT(packageEntry->fIdleCoreCount > 0); - ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); - - packageEntry->fIdleCoreCount--; - packageEntry->fIdleCores.Remove(&gCoreEntries[core]); - - if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { - // package wakes up - WriteSpinLocker _(gIdlePackageLock); - gIdlePackageList->Remove(packageEntry); - } - } -} - - -static inline int32 -choose_core(Thread* thread) -{ - ASSERT(!gSingleCore); - return sCurrentMode->choose_core(thread); -} - - -static inline int32 -choose_cpu(int32 core, Thread* thread, bool& rescheduleNeeded) -{ - SpinLocker cpuLocker(gCoreEntries[core].fCPULock); - CPUEntry* entry = gCPUPriorityHeaps[core].PeekMinimum(); - ASSERT(entry != NULL); - - int32 threadPriority = get_effective_priority(thread); - if (CPUHeap::GetKey(entry) < threadPriority) { - update_cpu_priority(entry->fCPUNumber, threadPriority); - rescheduleNeeded = true; - } else - rescheduleNeeded = false; - - return entry->fCPUNumber; -} - - -static bool -choose_core_and_cpu(Thread* thread, int32& targetCore, int32& targetCPU) -{ - bool rescheduleNeeded = false; - - if (targetCore == -1 && targetCPU != -1) - targetCore = gCPUToCore[targetCPU]; - else if (targetCore != -1 && targetCPU == -1) - targetCPU = choose_cpu(targetCore, thread, rescheduleNeeded); - else if (targetCore == -1 && targetCPU == -1) { - targetCore = choose_core(thread); - targetCPU = choose_cpu(targetCore, thread, rescheduleNeeded); - } - - ASSERT(targetCore >= 0 && targetCore < gCoreCount); - ASSERT(targetCPU >= 0 && targetCPU < smp_get_num_cpus()); - - return rescheduleNeeded; -} - - -static bool -should_rebalance(Thread* thread) -{ - ASSERT(!gSingleCore); - - return sCurrentMode->should_rebalance(thread); -} - - -static inline void -compute_cpu_load(int32 cpu) -{ - ASSERT(!gSingleCore); - - int oldLoad = compute_load(gCPUEntries[cpu].fMeasureTime, - gCPUEntries[cpu].fMeasureActiveTime, gCPUEntries[cpu].fLoad); - if (oldLoad < 0) - return; - - if (oldLoad != gCPUEntries[cpu].fLoad) { - int32 core = gCPUToCore[cpu]; - - int32 delta = gCPUEntries[cpu].fLoad - oldLoad; - atomic_add(&gCoreEntries[core].fLoad, delta); - - update_load_heaps(core); - } - - if (gCPUEntries[cpu].fLoad > kVeryHighLoad) - sCurrentMode->rebalance_irqs(false); -} - - -static inline void -compute_thread_load(Thread* thread) -{ - if (thread->scheduler_data->last_interrupt_time > 0) { - bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; - interruptTime -= thread->scheduler_data->last_interrupt_time; - thread->scheduler_data->measure_active_time -= interruptTime; - } - - compute_load(thread->scheduler_data->measure_time, - thread->scheduler_data->measure_active_time, - thread->scheduler_data->load); -} - - -static inline void -thread_goes_away(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - schedulerThreadData->last_interrupt_time = 0; - - schedulerThreadData->went_sleep = system_time(); - schedulerThreadData->went_sleep_active - = atomic_get64(&gCoreEntries[smp_get_current_cpu()].fActiveTime); - schedulerThreadData->went_sleep_count - = atomic_get(&gCoreEntries[smp_get_current_cpu()].fStarvationCounter); -} - - -static inline bool -should_cancel_penalty(Thread* thread) -{ - int32 core = thread->scheduler_data->previous_core; - - if (core < 0) - return false; - - return atomic_get(&gCoreEntries[core].fStarvationCounter) - != thread->scheduler_data->went_sleep_count - && system_time() - thread->scheduler_data->went_sleep - > sCurrentMode->base_quantum; + thread->scheduler_data->Dump(); } @@ -627,62 +145,45 @@ enqueue(Thread* thread, bool newOne) { ASSERT(thread != NULL); - thread->state = B_THREAD_READY; - - compute_thread_load(thread); - - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - schedulerThreadData->time_left = 0; - schedulerThreadData->went_sleep_count = 0; - int32 threadPriority = get_effective_priority(thread); + ThreadData* threadData = thread->scheduler_data; + int32 threadPriority = threadData->GetEffectivePriority(); T(EnqueueThread(thread, threadPriority)); - bool pinned = thread->pinned_to_cpu > 0; - int32 targetCPU = -1; - int32 targetCore = -1; - if (pinned) - targetCPU = thread->previous_cpu->cpu_num; - else if (gSingleCore) - targetCore = 0; - else if (schedulerThreadData->previous_core >= 0 - && (!newOne || !has_cache_expired(thread)) - && !should_rebalance(thread)) { - targetCore = schedulerThreadData->previous_core; + CPUEntry* targetCPU = NULL; + CoreEntry* targetCore = NULL; + if (thread->pinned_to_cpu > 0) { + ASSERT(thread->previous_cpu != NULL); + targetCPU = &gCPUEntries[thread->previous_cpu->cpu_num]; + } else if (gSingleCore) + targetCore = &gCoreEntries[0]; + else if (threadData->GetCore() != NULL + && (!newOne || !threadData->HasCacheExpired()) + && !threadData->ShouldRebalance()) { + targetCore = threadData->GetCore(); } - bool rescheduleNeeded = choose_core_and_cpu(thread, targetCore, targetCPU); - schedulerThreadData->previous_core = targetCore; + bool rescheduleNeeded = threadData->ChooseCoreAndCPU(targetCore, targetCPU); TRACE("enqueueing thread %ld with priority %ld on CPU %ld (core %ld)\n", - thread->id, threadPriority, targetCPU, targetCore); + thread->id, threadPriority, targetCPU->fCPUNumber, targetCore->fCoreID); - SpinLocker runQueueLocker(gCoreEntries[targetCore].fQueueLock); - thread->scheduler_data->enqueued = true; - if (pinned) - gPinnedRunQueues[targetCPU].PushBack(thread, threadPriority); - else { - gRunQueues[targetCore].PushBack(thread, threadPriority); - gCoreEntries[targetCore].fThreadList.Insert(thread->scheduler_data); - - atomic_add(&gCoreEntries[targetCore].fThreadCount, 1); - } - runQueueLocker.Unlock(); + threadData->Enqueue(); // notify listeners NotifySchedulerListeners(&SchedulerListener::ThreadEnqueuedInRunQueue, thread); - int32 heapPriority = CPUHeap::GetKey(&gCPUEntries[targetCPU]); - if (threadPriority > atomic_get(&gCPUEntries[targetCPU].fPriority) + int32 heapPriority = CPUPriorityHeap::GetKey(targetCPU); + if (threadPriority > atomic_get(&targetCPU->fPriority) && (threadPriority > heapPriority || (threadPriority == heapPriority && rescheduleNeeded))) { - if (targetCPU == smp_get_current_cpu()) - gCPU[targetCPU].invoke_scheduler = true; + if (targetCPU->fCPUNumber == smp_get_current_cpu()) + gCPU[targetCPU->fCPUNumber].invoke_scheduler = true; else { - smp_send_ici(targetCPU, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, - SMP_MSG_FLAG_ASYNC); + smp_send_ici(targetCPU->fCPUNumber, SMP_MSG_RESCHEDULE, 0, 0, 0, + NULL, SMP_MSG_FLAG_ASYNC); } } } @@ -699,43 +200,15 @@ scheduler_enqueue_in_run_queue(Thread *thread) TRACE("enqueueing new thread %ld with static priority %ld\n", thread->id, thread->priority); - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; + ThreadData* threadData = thread->scheduler_data; - int32 core = schedulerThreadData->previous_core; - if (core >= 0) { - if (should_cancel_penalty(thread)) - cancel_penalty(thread); - } + if (threadData->ShouldCancelPenalty()) + threadData->CancelPenalty(); enqueue(thread, true); } -static inline void -put_back(Thread* thread) -{ - compute_thread_load(thread); - thread->scheduler_data->went_sleep_count = -1; - - int32 core = gCPUToCore[smp_get_current_cpu()]; - - SpinLocker runQueueLocker(gCoreEntries[core].fQueueLock); - thread->scheduler_data->enqueued = true; - if (thread->pinned_to_cpu > 0) { - int32 pinnedCPU = thread->previous_cpu->cpu_num; - - ASSERT(pinnedCPU == smp_get_current_cpu()); - gPinnedRunQueues[pinnedCPU].PushFront(thread, - get_effective_priority(thread)); - } else { - ASSERT(thread->scheduler_data->previous_core == core); - - gRunQueues[core].PushFront(thread, get_effective_priority(thread)); - atomic_add(&gCoreEntries[core].fThreadCount, 1); - } -} - - /*! Sets the priority of a thread. */ int32 @@ -744,73 +217,44 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) InterruptsSpinLocker _(thread->scheduler_lock); SchedulerModeLocker modeLocker; + ThreadData* threadData = thread->scheduler_data; int32 oldPriority = thread->priority; TRACE("changing thread %ld priority to %ld (old: %ld, effective: %ld)\n", - thread->id, priority, oldPriority, get_effective_priority(thread)); + thread->id, priority, oldPriority, threadData->GetEffectivePriority()); - cancel_penalty(thread); + threadData->CancelPenalty(); if (priority == thread->priority) return thread->priority; - thread->priority = priority; - int32 previousCore = thread->scheduler_data->previous_core; - ASSERT(previousCore >= 0); - if (thread->state != B_THREAD_READY) { if (thread->state == B_THREAD_RUNNING) { - ASSERT(thread->previous_cpu != NULL); + ASSERT(threadData->GetCore() != NULL); - SpinLocker coreLocker(gCoreEntries[previousCore].fCPULock); + ASSERT(thread->cpu != NULL); + CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num]; - gCPUEntries[thread->cpu->cpu_num].fPriority = priority; - update_cpu_priority(thread->cpu->cpu_num, priority); + SpinLocker coreLocker(threadData->GetCore()->fCPULock); + cpu->fPriority = priority; + cpu->UpdatePriority(priority); } + return oldPriority; } // The thread is in the run queue. We need to remove it and re-insert it at // a new position. - bool pinned = thread->pinned_to_cpu > 0; - int32 previousCPU; - if (pinned) { - ASSERT(thread->previous_cpu != NULL); - previousCPU = thread->previous_cpu->cpu_num; - } + T(RemoveThread(thread)); - SpinLocker runQueueLocker(gCoreEntries[previousCore].fQueueLock); - - // the thread might have been already dequeued and is about to start - // running once we release its scheduler_lock, in such case we can not - // attempt to dequeue it - if (thread->scheduler_data->enqueued) { - T(RemoveThread(thread)); - - // notify listeners - NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, - thread); - - thread->scheduler_data->enqueued = false; - if (pinned) - gPinnedRunQueues[previousCPU].Remove(thread); - else { - gRunQueues[previousCore].Remove(thread); - - ASSERT(thread->scheduler_data->went_sleep_count < 1); - if (thread->scheduler_data->went_sleep_count == 0) { - gCoreEntries[previousCore].fThreadList.Remove( - thread->scheduler_data); - } - - atomic_add(&gCoreEntries[previousCore].fThreadCount, -1); - } - runQueueLocker.Unlock(); + // notify listeners + NotifySchedulerListeners(&SchedulerListener::ThreadRemovedFromRunQueue, + thread); + if (threadData->Dequeue()) enqueue(thread, true); - } return oldPriority; } @@ -841,212 +285,6 @@ reschedule_event(timer* /* unused */) } -static inline bool -quantum_ended(Thread* thread, bool wasPreempted, bool hasYielded) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - if (hasYielded) { - schedulerThreadData->time_left = 0; - return true; - } - - bigtime_t time_used = system_time() - schedulerThreadData->quantum_start; - schedulerThreadData->time_left -= time_used; - schedulerThreadData->time_left = max_c(0, schedulerThreadData->time_left); - - // too little time left, it's better make the next quantum a bit longer - if (wasPreempted - || schedulerThreadData->time_left <= sCurrentMode->minimal_quantum) { - - schedulerThreadData->stolen_time += schedulerThreadData->time_left; - schedulerThreadData->time_left = 0; - } - - return schedulerThreadData->time_left == 0; -} - - -static inline bigtime_t -quantum_linear_interpolation(bigtime_t maxQuantum, bigtime_t minQuantum, - int32 maxPriority, int32 minPriority, int32 priority) -{ - ASSERT(priority <= maxPriority); - ASSERT(priority >= minPriority); - - bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); - result /= maxPriority - minPriority; - return maxQuantum - result; -} - - -static inline bigtime_t -get_base_quantum(Thread* thread) -{ - int32 priority = get_effective_priority(thread); - - const bigtime_t kQuantum0 = sCurrentMode->base_quantum; - if (priority >= B_URGENT_DISPLAY_PRIORITY) - return kQuantum0; - - const bigtime_t kQuantum1 - = kQuantum0 * sCurrentMode->quantum_multipliers[0]; - if (priority > B_NORMAL_PRIORITY) { - return quantum_linear_interpolation(kQuantum1, kQuantum0, - B_URGENT_DISPLAY_PRIORITY, B_NORMAL_PRIORITY, priority); - } - - const bigtime_t kQuantum2 - = kQuantum0 * sCurrentMode->quantum_multipliers[1]; - return quantum_linear_interpolation(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, - B_IDLE_PRIORITY, priority); -} - - -static inline bigtime_t -compute_quantum(Thread* thread) -{ - scheduler_thread_data* schedulerThreadData = thread->scheduler_data; - - bigtime_t quantum; - if (schedulerThreadData->time_left != 0) - quantum = schedulerThreadData->time_left; - else - quantum = get_base_quantum(thread); - - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return quantum; - - quantum += schedulerThreadData->stolen_time; - schedulerThreadData->stolen_time = 0; - - ASSERT(schedulerThreadData->previous_core - == gCPUToCore[smp_get_current_cpu()]); - CoreEntry* core = &gCoreEntries[schedulerThreadData->previous_core]; - int32 threadCount = (core->fThreadCount + 1) / core->fCPUCount; - threadCount = max_c(threadCount, 1); - - quantum = max_c(min_c(sCurrentMode->maximum_latency / threadCount, quantum), - sCurrentMode->minimal_quantum); - - schedulerThreadData->time_left = quantum; - schedulerThreadData->quantum_start = system_time(); - return quantum; -} - - -static inline Thread* -choose_next_thread(int32 thisCPU, Thread* oldThread, bool putAtBack) -{ - int32 thisCore = gCPUToCore[thisCPU]; - - SpinLocker runQueueLocker(gCoreEntries[thisCore].fQueueLock); - - Thread* sharedThread = gRunQueues[thisCore].PeekMaximum(); - Thread* pinnedThread = gPinnedRunQueues[thisCPU].PeekMaximum(); - - ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); - - int32 pinnedPriority = -1; - if (pinnedThread != NULL) - pinnedPriority = get_effective_priority(pinnedThread); - - int32 sharedPriority = -1; - if (sharedThread != NULL) - sharedPriority = get_effective_priority(sharedThread); - - int32 oldPriority = -1; - if (oldThread != NULL) - oldPriority = get_effective_priority(oldThread); - - int32 rest = max_c(pinnedPriority, sharedPriority); - if (oldPriority > rest || (!putAtBack && oldPriority == rest)) { - ASSERT(!oldThread->scheduler_data->enqueued); - return oldThread; - } - - if (sharedPriority > pinnedPriority) { - ASSERT(sharedThread->scheduler_data->enqueued); - sharedThread->scheduler_data->enqueued = false; - - gRunQueues[thisCore].Remove(sharedThread); - if (thread_is_idle_thread(sharedThread) - || gCoreEntries[thisCore].fThreadList.Head() - == sharedThread->scheduler_data) { - atomic_add(&gCoreEntries[thisCore].fStarvationCounter, 1); - } - - if (sharedThread->scheduler_data->went_sleep_count == 0) { - gCoreEntries[thisCore].fThreadList.Remove( - sharedThread->scheduler_data); - } - - atomic_add(&gCoreEntries[thisCore].fThreadCount, -1); - return sharedThread; - } - - ASSERT(pinnedThread->scheduler_data->enqueued); - pinnedThread->scheduler_data->enqueued = false; - - gPinnedRunQueues[thisCPU].Remove(pinnedThread); - return pinnedThread; -} - - -static inline void -track_cpu_activity(Thread* oldThread, Thread* nextThread, int32 thisCore) -{ - if (!thread_is_idle_thread(oldThread)) { - bigtime_t active - = (oldThread->kernel_time - oldThread->cpu->last_kernel_time) - + (oldThread->user_time - oldThread->cpu->last_user_time); - - atomic_add64(&oldThread->cpu->active_time, active); - oldThread->scheduler_data->measure_active_time += active; - - gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - atomic_add64(&gCoreEntries[thisCore].fActiveTime, active); - } - - compute_thread_load(oldThread); - compute_thread_load(nextThread); - if (!gSingleCore && !gCPU[smp_get_current_cpu()].disabled) - compute_cpu_load(smp_get_current_cpu()); - - if (!thread_is_idle_thread(nextThread)) { - oldThread->cpu->last_kernel_time = nextThread->kernel_time; - oldThread->cpu->last_user_time = nextThread->user_time; - } -} - - -static inline void -update_cpu_performance(Thread* thread, int32 thisCore) -{ - int32 load = max_c(thread->scheduler_data->load, - get_core_load(&gCoreEntries[thisCore])); - load = min_c(max_c(load, 0), kMaxLoad); - - if (load < kTargetLoad) { - int32 delta = kTargetLoad - load; - - delta *= kTargetLoad; - delta /= kCPUPerformanceScaleMax; - - decrease_cpu_performance(delta); - } else { - bool allowBoost = !sCurrentMode->avoid_boost; - allowBoost = allowBoost || thread->scheduler_data->priority_penalty > 0; - - int32 delta = load - kTargetLoad; - delta *= kMaxLoad - kTargetLoad; - delta /= kCPUPerformanceScaleMax; - - increase_cpu_performance(delta, allowBoost); - } -} - - static inline void stop_cpu_timers(Thread* fromThread, Thread* toThread) { @@ -1170,18 +408,19 @@ reschedule(int32 nextState) Thread* oldThread = thread_get_current_thread(); int32 thisCPU = smp_get_current_cpu(); - int32 thisCore = gCPUToCore[thisCPU]; + + CPUEntry* cpu = &gCPUEntries[thisCPU]; + CoreEntry* core = CoreEntry::GetCore(thisCPU); TRACE("reschedule(): cpu %ld, current thread = %ld\n", thisCPU, oldThread->id); oldThread->state = nextState; - scheduler_thread_data* schedulerOldThreadData = oldThread->scheduler_data; + ThreadData* oldThreadData = oldThread->scheduler_data; // return time spent in interrupts - schedulerOldThreadData->stolen_time - += gCPU[thisCPU].interrupt_time - - schedulerOldThreadData->last_interrupt_time; + oldThreadData->fStolenTime + += gCPU[thisCPU].interrupt_time - oldThreadData->fLastInterruptTime; bool enqueueOldThread = false; bool putOldThreadAtBack = false; @@ -1190,16 +429,16 @@ reschedule(int32 nextState) case B_THREAD_READY: enqueueOldThread = true; - if (quantum_ended(oldThread, oldThread->cpu->preempted, + if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, oldThread->has_yielded)) { - increase_penalty(oldThread); + oldThreadData->IncreasePenalty(); TRACE("enqueueing thread %ld into run queue priority = %ld\n", - oldThread->id, get_effective_priority(oldThread)); + oldThread->id, oldThreadData->GetEffectivePriority()); putOldThreadAtBack = true; } else { TRACE("putting thread %ld back in run queue priority = %ld\n", - oldThread->id, get_effective_priority(oldThread)); + oldThread->id, oldThreadData->GetEffectivePriority()); putOldThreadAtBack = false; } @@ -1207,8 +446,8 @@ reschedule(int32 nextState) case THREAD_STATE_FREE_ON_RESCHED: break; default: - increase_penalty(oldThread); - thread_goes_away(oldThread); + oldThreadData->IncreasePenalty(); + oldThreadData->GoesAway(); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, nextState); break; @@ -1217,33 +456,33 @@ reschedule(int32 nextState) oldThread->has_yielded = false; // select thread with the biggest priority and enqueue back the old thread - Thread* nextThread; + ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { if (!thread_is_idle_thread(oldThread)) { - SpinLocker runQueueLocker(gCoreEntries[thisCore].fQueueLock); + SpinLocker runQueueLocker(core->fQueueLock); - nextThread = gPinnedRunQueues[thisCPU].GetHead(B_IDLE_PRIORITY); - gPinnedRunQueues[thisCPU].Remove(nextThread); - nextThread->scheduler_data->enqueued = false; + nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY); + cpu->fRunQueue.Remove(nextThreadData); + nextThreadData->fEnqueued = false; putOldThreadAtBack = oldThread->pinned_to_cpu == 0; } else - nextThread = oldThread; + nextThreadData = oldThreadData; } else { - nextThread - = choose_next_thread(thisCPU, enqueueOldThread ? oldThread : NULL, + nextThreadData + = cpu->ChooseNextThread(enqueueOldThread ? oldThreadData : NULL, putOldThreadAtBack); } - atomic_set(&gCPUEntries[thisCPU].fPriority, - get_effective_priority(nextThread)); + Thread* nextThread = nextThreadData->GetThread(); + atomic_set(&cpu->fPriority, nextThreadData->GetEffectivePriority()); if (nextThread != oldThread) { if (enqueueOldThread) { if (putOldThreadAtBack) enqueue(oldThread, false); else - put_back(oldThread); + oldThreadData->PutBack(); } acquire_spinlock(&nextThread->scheduler_lock); @@ -1258,28 +497,20 @@ reschedule(int32 nextState) NotifySchedulerListeners(&SchedulerListener::ThreadScheduled, oldThread, nextThread); - // update CPU heap - if (!gCPU[thisCPU].disabled) { - SpinLocker coreLocker(gCoreEntries[thisCore].fCPULock); - update_cpu_priority(thisCPU, get_effective_priority(nextThread)); - } - + ASSERT(nextThreadData->GetCore() == core); nextThread->state = B_THREAD_RUNNING; - ASSERT(nextThread->scheduler_data->previous_core == thisCore); + // update CPU heap + if (!gCPU[thisCPU].disabled) { + SpinLocker coreLocker(core->fCPULock); + cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); + } // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); // track CPU activity - track_cpu_activity(oldThread, nextThread, thisCore); - - // start counting time spent in interrupts - nextThread->scheduler_data->last_interrupt_time - = gCPU[thisCPU].interrupt_time; - - if (!thread_is_idle_thread(nextThread)) - update_cpu_performance(nextThread, thisCore); + cpu->TrackActivity(oldThreadData, nextThreadData); if (nextThread != oldThread || oldThread->cpu->preempted) { timer* quantumTimer = &oldThread->cpu->quantum_timer; @@ -1288,13 +519,13 @@ reschedule(int32 nextState) oldThread->cpu->preempted = false; if (!thread_is_idle_thread(nextThread)) { - bigtime_t quantum = compute_quantum(nextThread); + bigtime_t quantum = nextThreadData->ComputeQuantum(); add_timer(quantumTimer, &reschedule_event, quantum, B_ONE_SHOT_RELATIVE_TIMER); } else { - nextThread->scheduler_data->quantum_start = system_time(); + nextThreadData->fQuantumStart = system_time(); - sCurrentMode->rebalance_irqs(true); + gCurrentMode->rebalance_irqs(true); } modeLocker.Unlock(); @@ -1324,7 +555,7 @@ scheduler_reschedule(int32 nextState) status_t scheduler_on_thread_create(Thread* thread, bool idleThread) { - thread->scheduler_data = new (std::nothrow)scheduler_thread_data; + thread->scheduler_data = new(std::nothrow) ThreadData(thread); if (thread->scheduler_data == NULL) return B_NO_MEMORY; return B_OK; @@ -1334,17 +565,16 @@ scheduler_on_thread_create(Thread* thread, bool idleThread) void scheduler_on_thread_init(Thread* thread) { - thread->scheduler_data->Init(); - if (thread_is_idle_thread(thread)) { static int32 sIdleThreadsID; - int32 cpu = atomic_add(&sIdleThreadsID, 1); + int32 cpuID = atomic_add(&sIdleThreadsID, 1); - thread->previous_cpu = &gCPU[cpu]; + thread->previous_cpu = &gCPU[cpuID]; thread->pinned_to_cpu = 1; - thread->scheduler_data->previous_core = gCPUToCore[cpu]; - } + thread->scheduler_data->Init(CoreEntry::GetCore(cpuID)); + } else + thread->scheduler_data->Init(); } @@ -1367,22 +597,6 @@ scheduler_start(void) } -static inline void -acquire_big_scheduler_lock(void) -{ - for (int32 i = 0; i < smp_get_num_cpus(); i++) - acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); -} - - -static inline void -release_big_scheduler_lock(void) -{ - for (int32 i = 0; i < smp_get_num_cpus(); i++) - release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock); -} - - status_t scheduler_set_operation_mode(scheduler_mode mode) { @@ -1393,14 +607,11 @@ scheduler_set_operation_mode(scheduler_mode mode) dprintf("scheduler: switching to %s mode\n", sSchedulerModes[mode]->name); - InterruptsLocker _; - acquire_big_scheduler_lock(); + InterruptsBigSchedulerLocker _; - sCurrentModeID = mode; - sCurrentMode = sSchedulerModes[mode]; - sCurrentMode->switch_to_mode(); - - release_big_scheduler_lock(); + gCurrentModeID = mode; + gCurrentMode = sSchedulerModes[mode]; + gCurrentMode->switch_to_mode(); return B_OK; } @@ -1409,38 +620,38 @@ scheduler_set_operation_mode(scheduler_mode mode) static void unassign_thread(Thread* thread, void* data) { - int32 core = *(int32*)data; + CoreEntry* core = static_cast(data); - if (thread->scheduler_data->previous_core == core + if (thread->scheduler_data->GetCore() == core && thread->pinned_to_cpu == 0) { - thread->scheduler_data->previous_core = -1; + thread->scheduler_data->UnassignCore(); } } void -scheduler_set_cpu_enabled(int32 cpu, bool enabled) +scheduler_set_cpu_enabled(int32 cpuID, bool enabled) { dprintf("scheduler: %s CPU %" B_PRId32 "\n", - enabled ? "enabling" : "disabling", cpu); + enabled ? "enabling" : "disabling", cpuID); - InterruptsLocker _; - acquire_big_scheduler_lock(); + InterruptsBigSchedulerLocker _; - gCPU[cpu].disabled = !enabled; + gCPU[cpuID].disabled = !enabled; - sCurrentMode->set_cpu_enabled(cpu, enabled); + gCurrentMode->set_cpu_enabled(cpuID, enabled); - CoreEntry* core = &gCoreEntries[gCPUToCore[cpu]]; - PackageEntry* package = &gPackageEntries[gCPUToPackage[cpu]]; + CPUEntry* cpu = &gCPUEntries[cpuID]; + CoreEntry* core = cpu->fCore; + PackageEntry* package = core->fPackage; int32 oldCPUCount = core->fCPUCount; ASSERT(oldCPUCount >= 0); if (enabled) core->fCPUCount++; else { - gCPUEntries[cpu].fPriority = B_IDLE_PRIORITY; - update_cpu_priority(cpu, B_IDLE_PRIORITY); + cpu->fPriority = B_IDLE_PRIORITY; + cpu->UpdatePriority(B_IDLE_PRIORITY); core->fCPUCount--; } @@ -1449,13 +660,13 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) ASSERT(!enabled); if (core->fHighLoad) { - gCoreHighLoadHeap->ModifyKey(core, -1); - ASSERT(gCoreHighLoadHeap->PeekMinimum() == core); - gCoreHighLoadHeap->RemoveMinimum(); + gCoreHighLoadHeap.ModifyKey(core, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == core); + gCoreHighLoadHeap.RemoveMinimum(); } else { - gCoreLoadHeap->ModifyKey(core, -1); - ASSERT(gCoreLoadHeap->PeekMinimum() == core); - gCoreLoadHeap->RemoveMinimum(); + gCoreLoadHeap.ModifyKey(core, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == core); + gCoreLoadHeap.RemoveMinimum(); } package->fIdleCores.Remove(core); @@ -1463,63 +674,58 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) package->fCoreCount--; if (package->fCoreCount == 0) - gIdlePackageList->Remove(package); + gIdlePackageList.Remove(package); // get rid of threads thread_map(unassign_thread, &core->fCoreID); core->fThreadCount = 0; - while (gRunQueues[core->fCoreID].PeekMaximum() != NULL) { - Thread* thread = gRunQueues[core->fCoreID].PeekMaximum(); - gRunQueues[core->fCoreID].Remove(thread); - thread->scheduler_data->enqueued = false; + while (core->fRunQueue.PeekMaximum() != NULL) { + ThreadData* threadData = core->fRunQueue.PeekMaximum(); - if (thread->scheduler_data->went_sleep_count == 0) { - gCoreEntries[core->fCoreID].fThreadList.Remove( - thread->scheduler_data); - thread->scheduler_data->went_sleep_count = -1;; + core->fRunQueue.Remove(threadData); + threadData->fEnqueued = false; + + if (threadData->fWentSleepCount == 0) { + core->fThreadList.Remove(threadData); + threadData->fWentSleepCount = -1; } - ASSERT(thread->scheduler_data->previous_core == -1); - enqueue(thread, false); + ASSERT(threadData->GetCore() == NULL); + enqueue(threadData->GetThread(), false); } } else if (oldCPUCount == 0) { // core has been reenabled ASSERT(enabled); - gCPUEntries[cpu].fLoad = 0; + cpu->fLoad = 0; core->fLoad = 0; core->fHighLoad = false; - gCoreLoadHeap->Insert(core, 0); + gCoreLoadHeap.Insert(core, 0); package->fCoreCount++; package->fIdleCoreCount++; package->fIdleCores.Add(core); if (package->fCoreCount == 1) - gIdlePackageList->Add(package); + gIdlePackageList.Add(package); } if (enabled) { - gCPUPriorityHeaps[core->fCoreID].Insert(&gCPUEntries[cpu], - B_IDLE_PRIORITY); - gCPUEntries[cpu].fLoad = 0; + core->fCPUHeap.Insert(cpu, B_IDLE_PRIORITY); + cpu->fLoad = 0; } else { - gCPUPriorityHeaps[core->fCoreID].ModifyKey(&gCPUEntries[cpu], - THREAD_MAX_SET_PRIORITY + 1); - ASSERT(gCPUPriorityHeaps[core->fCoreID].PeekMaximum() - == &gCPUEntries[cpu]); - gCPUPriorityHeaps[core->fCoreID].RemoveMaximum(); + core->fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); + ASSERT(core->fCPUHeap.PeekMaximum() == cpu); + core->fCPUHeap.RemoveMaximum(); - ASSERT(gCPUEntries[cpu].fLoad >= 0 - && gCPUEntries[cpu].fLoad <= kMaxLoad); - - core->fLoad -= gCPUEntries[cpu].fLoad; + ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); + core->fLoad -= cpu->fLoad; ASSERT(core->fLoad >= 0); } if (!enabled) { - cpu_ent* entry = &gCPU[cpu]; + cpu_ent* entry = &gCPU[cpuID]; // get rid of irqs SpinLocker locker(entry->irqs_lock); @@ -1536,13 +742,11 @@ scheduler_set_cpu_enabled(int32 cpu, bool enabled) locker.Unlock(); // don't wait until the thread quantum ends - if (smp_get_current_cpu() != cpu) { - smp_send_ici(cpu, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, + if (smp_get_current_cpu() != cpuID) { + smp_send_ici(cpuID, SMP_MSG_RESCHEDULE, 0, 0, 0, NULL, SMP_MSG_FLAG_ASYNC); } } - - release_big_scheduler_lock(); } @@ -1551,8 +755,8 @@ traverse_topology_tree(cpu_topology_node* node, int packageID, int coreID) { switch (node->level) { case CPU_TOPOLOGY_SMT: - gCPUToCore[node->id] = coreID; - gCPUToPackage[node->id] = packageID; + sCPUToCore[node->id] = coreID; + sCPUToPackage[node->id] = packageID; return; case CPU_TOPOLOGY_CORE: @@ -1577,15 +781,15 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) { cpuCount = smp_get_num_cpus(); - gCPUToCore = new(std::nothrow) int32[cpuCount]; - if (gCPUToCore == NULL) + sCPUToCore = new(std::nothrow) int32[cpuCount]; + if (sCPUToCore == NULL) return B_NO_MEMORY; - ArrayDeleter cpuToCoreDeleter(gCPUToCore); + ArrayDeleter cpuToCoreDeleter(sCPUToCore); - gCPUToPackage = new(std::nothrow) int32[cpuCount]; - if (gCPUToPackage == NULL) + sCPUToPackage = new(std::nothrow) int32[cpuCount]; + if (sCPUToPackage == NULL) return B_NO_MEMORY; - ArrayDeleter cpuToPackageDeleter(gCPUToPackage); + ArrayDeleter cpuToPackageDeleter(sCPUToPackage); coreCount = 0; for (int32 i = 0; i < cpuCount; i++) { @@ -1610,23 +814,6 @@ build_topology_mappings(int32& cpuCount, int32& coreCount, int32& packageCount) } -static status_t -create_debug_heaps() -{ - sDebugCPUHeap = new(std::nothrow) CPUHeap(smp_get_num_cpus()); - if (sDebugCPUHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter cpuDeleter(sDebugCPUHeap); - - sDebugCoreHeap = new(std::nothrow) CoreLoadHeap(smp_get_num_cpus()); - if (sDebugCoreHeap == NULL) - return B_NO_MEMORY; - - cpuDeleter.Detach(); - return B_OK; -} - - static status_t init() { @@ -1636,120 +823,68 @@ init() packageCount); if (result != B_OK) return result; + gCoreCount = coreCount; gSingleCore = coreCount == 1; gPackageCount = packageCount; - // create package heap and idle package stack + gCPUEntries = new(std::nothrow) CPUEntry[cpuCount]; + if (gCPUEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter cpuEntriesDeleter(gCPUEntries); + + gCoreEntries = new(std::nothrow) CoreEntry[coreCount]; + if (gCoreEntries == NULL) + return B_NO_MEMORY; + ArrayDeleter coreEntriesDeleter(gCoreEntries); + gPackageEntries = new(std::nothrow) PackageEntry[packageCount]; if (gPackageEntries == NULL) return B_NO_MEMORY; ArrayDeleter packageEntriesDeleter(gPackageEntries); - gIdlePackageList = new(std::nothrow) IdlePackageList; - if (gIdlePackageList == NULL) - return B_NO_MEMORY; - ObjectDeleter packageListDeleter(gIdlePackageList); + new(&gCoreLoadHeap) CoreLoadHeap(coreCount); + new(&gCoreHighLoadHeap) CoreLoadHeap(coreCount); + + new(&gIdlePackageList) IdlePackageList; for (int32 i = 0; i < packageCount; i++) { gPackageEntries[i].fPackageID = i; - gPackageEntries[i].fIdleCoreCount = coreCount / packageCount; gPackageEntries[i].fCoreCount = coreCount / packageCount; - gIdlePackageList->Insert(&gPackageEntries[i]); + gIdlePackageList.Insert(&gPackageEntries[i]); } - // create logical processor and core heaps - gCPUEntries = new CPUEntry[cpuCount]; - if (gCPUEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuEntriesDeleter(gCPUEntries); - - gCoreEntries = new CoreEntry[coreCount]; - if (gCoreEntries == NULL) - return B_NO_MEMORY; - ArrayDeleter coreEntriesDeleter(gCoreEntries); - - gCoreLoadHeap = new CoreLoadHeap; - if (gCoreLoadHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter coreLoadHeapDeleter(gCoreLoadHeap); - - gCoreHighLoadHeap = new CoreLoadHeap(coreCount); - if (gCoreHighLoadHeap == NULL) - return B_NO_MEMORY; - ObjectDeleter coreHighLoadHeapDeleter(gCoreHighLoadHeap); - for (int32 i = 0; i < coreCount; i++) { gCoreEntries[i].fCoreID = i; gCoreEntries[i].fCPUCount = cpuCount / coreCount; - result = gCoreLoadHeap->Insert(&gCoreEntries[i], 0); + result = gCoreLoadHeap.Insert(&gCoreEntries[i], 0); if (result != B_OK) return result; } - gCPUPriorityHeaps = new CPUHeap[coreCount]; - if (gCPUPriorityHeaps == NULL) - return B_NO_MEMORY; - ArrayDeleter cpuPriorityHeapDeleter(gCPUPriorityHeaps); - for (int32 i = 0; i < cpuCount; i++) { + CoreEntry* core = &gCoreEntries[sCPUToCore[i]]; + PackageEntry* package = &gPackageEntries[sCPUToPackage[i]]; + gCPUEntries[i].fCPUNumber = i; + gCPUEntries[i].fCore = core; + core->fPackage = package; - int32 core = gCPUToCore[i]; + if (core->fCPUHeap.PeekMaximum() == NULL) { + package->fIdleCoreCount++; + package->fIdleCores.Insert(core); + } - int32 package = gCPUToPackage[i]; - if (gCPUPriorityHeaps[core].PeekMaximum() == NULL) - gPackageEntries[package].fIdleCores.Insert(&gCoreEntries[core]); - - result - = gCPUPriorityHeaps[core].Insert(&gCPUEntries[i], B_IDLE_PRIORITY); + result = core->fCPUHeap.Insert(&gCPUEntries[i], B_IDLE_PRIORITY); if (result != B_OK) return result; } - // create per-logical processor run queues for pinned threads - TRACE("scheduler_init(): creating %" B_PRId32 " per-cpu queue%s\n", - cpuCount, cpuCount != 1 ? "s" : ""); - - gPinnedRunQueues = new(std::nothrow) ThreadRunQueue[cpuCount]; - if (gPinnedRunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter pinnedRunQueuesDeleter(gPinnedRunQueues); - for (int i = 0; i < cpuCount; i++) { - result = gPinnedRunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - - // create per-core run queues - TRACE("scheduler_init(): creating %" B_PRId32 " per-core queue%s\n", - coreCount, coreCount != 1 ? "s" : ""); - - gRunQueues = new(std::nothrow) ThreadRunQueue[coreCount]; - if (gRunQueues == NULL) - return B_NO_MEMORY; - ArrayDeleter runQueuesDeleter(gRunQueues); - for (int i = 0; i < coreCount; i++) { - result = gRunQueues[i].GetInitStatus(); - if (result != B_OK) - return result; - } - - // create temporary heaps for debugging commands - result = create_debug_heaps(); - if (result != B_OK) - return result; - - runQueuesDeleter.Detach(); - pinnedRunQueuesDeleter.Detach(); - coreHighLoadHeapDeleter.Detach(); - coreLoadHeapDeleter.Detach(); - cpuPriorityHeapDeleter.Detach(); + packageEntriesDeleter.Detach(); coreEntriesDeleter.Detach(); cpuEntriesDeleter.Detach(); - packageEntriesDeleter.Detach(); - packageListDeleter.Detach(); + return B_OK; } @@ -1768,15 +903,7 @@ scheduler_init(void) scheduler_set_operation_mode(SCHEDULER_MODE_LOW_LATENCY); - add_debugger_command_etc("run_queue", &dump_run_queue, - "List threads in run queue", "\nLists threads in run queue", 0); - if (!gSingleCore) { - add_debugger_command_etc("cpu_heap", &dump_cpu_heap, - "List CPUs in CPU priority heap", - "\nList CPUs in CPU priority heap", 0); - add_debugger_command_etc("idle_cores", &dump_idle_cores, - "List idle cores", "\nList idle cores", 0); - } + init_debug_commands(); #if SCHEDULER_TRACING add_debugger_command_etc("scheduler", &cmd_scheduler, @@ -1846,22 +973,23 @@ _user_estimate_max_scheduling_latency(thread_id id) } BReference threadReference(thread, true); - int32 core = thread->scheduler_data->previous_core; - if (core == -1) - core = get_random() % gCoreCount; + ThreadData* threadData = thread->scheduler_data; + CoreEntry* core = threadData->GetCore(); + if (core == NULL) + core = &gCoreEntries[get_random() % gCoreCount]; - int32 threadCount = gCoreEntries[core].fThreadCount; - if (gCoreEntries[core].fCPUCount > 0) - threadCount /= gCoreEntries[core].fCPUCount; + int32 threadCount = core->fThreadCount; + if (core->fCPUCount > 0) + threadCount /= core->fCPUCount; - if (get_effective_priority(thread) > 0) { + if (threadData->GetEffectivePriority() > 0) { threadCount -= threadCount * THREAD_MAX_SET_PRIORITY - / get_effective_priority(thread); + / threadData->GetEffectivePriority(); } - return min_c(max_c(threadCount * sCurrentMode->base_quantum, - sCurrentMode->minimal_quantum), - sCurrentMode->maximum_latency); + return std::min(std::max(threadCount * gCurrentMode->base_quantum, + gCurrentMode->minimal_quantum), + gCurrentMode->maximum_latency); } @@ -1875,6 +1003,6 @@ _user_set_scheduler_mode(int32 mode) int32 _user_get_scheduler_mode(void) { - return sCurrentModeID; + return gCurrentModeID; } diff --git a/src/system/kernel/scheduler/scheduler_common.h b/src/system/kernel/scheduler/scheduler_common.h index 3069fffa5a..75490dd30b 100644 --- a/src/system/kernel/scheduler/scheduler_common.h +++ b/src/system/kernel/scheduler/scheduler_common.h @@ -7,6 +7,8 @@ #define KERNEL_SCHEDULER_COMMON_H +#include + #include #include #include @@ -29,6 +31,9 @@ namespace Scheduler { +struct CPUEntry; +struct CoreEntry; + const int kLowLoad = kMaxLoad * 20 / 100; const int kTargetLoad = kMaxLoad * 55 / 100; const int kHighLoad = kMaxLoad * 70 / 100; @@ -39,179 +44,12 @@ const int kLoadDifference = kMaxLoad * 20 / 100; extern bool gSingleCore; -// Heaps in sCPUPriorityHeaps are used for load balancing on a core the logical -// processors in the heap belong to. Since there are no cache affinity issues -// at this level and the run queue is shared among all logical processors on -// the core the only real concern is to make lower priority threads give way to -// the higher priority threads. -struct CPUEntry : public MinMaxHeapLinkImpl { - CPUEntry(); - int32 fCPUNumber; - - int32 fPriority; - - bigtime_t fMeasureActiveTime; - bigtime_t fMeasureTime; - - int32 fLoad; - - rw_spinlock fSchedulerModeLock; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap CPUHeap CACHE_LINE_ALIGN; - -extern CPUEntry* gCPUEntries; -extern CPUHeap* gCPUPriorityHeaps; - -struct CoreEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { - CoreEntry(); - - int32 fCoreID; - - int32 fCPUCount; - - spinlock fCPULock; - spinlock fQueueLock; - - int32 fStarvationCounter; - - int32 fThreadCount; - DoublyLinkedList fThreadList; - - bigtime_t fActiveTime; - - int32 fLoad; - bool fHighLoad; -} CACHE_LINE_ALIGN; -typedef MinMaxHeap CoreLoadHeap; - -extern CoreEntry* gCoreEntries; -extern CoreLoadHeap* gCoreLoadHeap; -extern CoreLoadHeap* gCoreHighLoadHeap; -extern rw_spinlock gCoreHeapsLock; -extern int32 gCoreCount; - -// gPackageEntries are used to decide which core should be woken up from the -// idle state. When aiming for performance we should use as many packages as -// possible with as little cores active in each package as possible (so that the -// package can enter any boost mode if it has one and the active core have more -// of the shared cache for themselves. If power saving is the main priority we -// should keep active cores on as little packages as possible (so that other -// packages can go to the deep state of sleep). The heap stores only packages -// with at least one core active and one core idle. The packages with all cores -// idle are stored in sPackageIdleList (in LIFO manner). -struct PackageEntry : public DoublyLinkedListLinkImpl { - PackageEntry(); - - int32 fPackageID; - - rw_spinlock fCoreLock; - - DoublyLinkedList fIdleCores; - int32 fIdleCoreCount; - - int32 fCoreCount; -} CACHE_LINE_ALIGN; -typedef DoublyLinkedList IdlePackageList; - -extern PackageEntry* gPackageEntries; -extern IdlePackageList* gIdlePackageList; -extern rw_spinlock gIdlePackageLock; -extern int32 gPackageCount; - -// The run queues. Holds the threads ready to run ordered by priority. -// One queue per schedulable target per core. Additionally, each -// logical processor has its sPinnedRunQueues used for scheduling -// pinned threads. -typedef RunQueue CACHE_LINE_ALIGN - ThreadRunQueue; - -extern ThreadRunQueue* gRunQueues; -extern ThreadRunQueue* gPinnedRunQueues; - -// Since CPU IDs used internally by the kernel bear no relation to the actual -// CPU topology the following arrays are used to efficiently get the core -// and the package that CPU in question belongs to. -extern int32* gCPUToCore; -extern int32* gCPUToPackage; +void init_debug_commands(void); } // namespace Scheduler -struct scheduler_thread_data : - public DoublyLinkedListLinkImpl { - inline scheduler_thread_data(); - void Init(); - - int32 priority_penalty; - int32 additional_penalty; - - bigtime_t time_left; - bigtime_t stolen_time; - bigtime_t quantum_start; - bigtime_t last_interrupt_time; - - bigtime_t measure_active_time; - bigtime_t measure_time; - int32 load; - - bigtime_t went_sleep; - bigtime_t went_sleep_active; - int32 went_sleep_count; - - int32 previous_core; - - bool enqueued; -}; - - -static inline int32 -get_core_load(struct Scheduler::CoreEntry* core) -{ - return core->fLoad / core->fCPUCount; -} - - -static inline int32 -get_minimal_priority(Thread* thread) -{ - return max_c(min_c(thread->priority, 25) / 5, 1); -} - - -static inline int32 -get_thread_penalty(Thread* thread) -{ - int32 penalty = thread->scheduler_data->priority_penalty; - - const int kMinimalPriority = get_minimal_priority(thread); - if (kMinimalPriority > 0) { - penalty - += thread->scheduler_data->additional_penalty % kMinimalPriority; - } - - return penalty; -} - - -static inline int32 -get_effective_priority(Thread* thread) -{ - if (thread->priority == B_IDLE_PRIORITY) - return thread->priority; - if (thread->priority >= B_FIRST_REAL_TIME_PRIORITY) - return thread->priority; - - int32 effectivePriority = thread->priority; - effectivePriority -= get_thread_penalty(thread); - - ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); - ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); - - return effectivePriority; -} - - #endif // KERNEL_SCHEDULER_COMMON_H + diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp new file mode 100644 index 0000000000..c78d62a149 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -0,0 +1,466 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + + +#include "scheduler_cpu.h" + +#include + +#include + +#include "scheduler_thread.h" + + +using namespace Scheduler; + + +static CPUPriorityHeap sDebugCPUHeap; +static CoreLoadHeap sDebugCoreHeap; + + +void +ThreadRunQueue::Dump() const +{ + ThreadRunQueue::ConstIterator iterator = GetConstIterator(); + if (!iterator.HasNext()) + kprintf("Run queue is empty.\n"); + else { + kprintf("thread id priority penalty name\n"); + while (iterator.HasNext()) { + ThreadData* threadData = iterator.Next(); + Thread* thread = threadData->GetThread(); + + kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", + thread, thread->id, thread->priority, + threadData->GetEffectivePriority(), thread->name); + } + } +} + + +CPUEntry::CPUEntry() + : + fPriority(B_IDLE_PRIORITY), + fLoad(0), + fMeasureActiveTime(0), + fMeasureTime(0) +{ + B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock); +} + + +void +CPUEntry::UpdatePriority(int32 priority) +{ + int32 corePriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); + fCore->fCPUHeap.ModifyKey(this, priority); + + if (gSingleCore) + return; + + int32 maxPriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); + if (corePriority == maxPriority) + return; + + PackageEntry* packageEntry = fCore->fPackage; + if (maxPriority == B_IDLE_PRIORITY) { + WriteSpinLocker _(packageEntry->fCoreLock); + + // core goes idle + ASSERT(packageEntry->fIdleCoreCount >= 0); + ASSERT(packageEntry->fIdleCoreCount < packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount++; + packageEntry->fIdleCores.Add(fCore); + + if (packageEntry->fIdleCoreCount == packageEntry->fCoreCount) { + // package goes idle + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Add(packageEntry); + } + } else if (corePriority == B_IDLE_PRIORITY) { + WriteSpinLocker _(packageEntry->fCoreLock); + + // core wakes up + ASSERT(packageEntry->fIdleCoreCount > 0); + ASSERT(packageEntry->fIdleCoreCount <= packageEntry->fCoreCount); + + packageEntry->fIdleCoreCount--; + packageEntry->fIdleCores.Remove(fCore); + + if (packageEntry->fIdleCoreCount + 1 == packageEntry->fCoreCount) { + // package wakes up + WriteSpinLocker _(gIdlePackageLock); + gIdlePackageList.Remove(packageEntry); + } + } +} + + +void +CPUEntry::ComputeLoad() +{ + ASSERT(!gSingleCore); + ASSERT(fCPUNumber == smp_get_current_cpu()); + + int oldLoad = compute_load(fMeasureTime, fMeasureActiveTime, fLoad); + if (oldLoad < 0) + return; + + if (oldLoad != fLoad) { + int32 delta = fLoad - oldLoad; + atomic_add(&fCore->fLoad, delta); + + fCore->UpdateLoad(); + } + + if (fLoad > kVeryHighLoad) + gCurrentMode->rebalance_irqs(false); +} + + +ThreadData* +CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) +{ + SpinLocker runQueueLocker(fCore->fQueueLock); + + ThreadData* sharedThread = fCore->fRunQueue.PeekMaximum(); + ThreadData* pinnedThread = fRunQueue.PeekMaximum(); + + ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); + + int32 pinnedPriority = -1; + if (pinnedThread != NULL) + pinnedPriority = pinnedThread->GetEffectivePriority(); + + int32 sharedPriority = -1; + if (sharedThread != NULL) + sharedPriority = sharedThread->GetEffectivePriority(); + + int32 oldPriority = -1; + if (oldThread != NULL) + oldPriority = oldThread->GetEffectivePriority(); + + int32 rest = std::max(pinnedPriority, sharedPriority); + if (oldPriority > rest || (!putAtBack && oldPriority == rest)) + return oldThread; + + if (sharedPriority > pinnedPriority) { + sharedThread->fEnqueued = false; + + fCore->fRunQueue.Remove(sharedThread); + if (thread_is_idle_thread(sharedThread->GetThread()) + || fCore->fThreadList.Head() == sharedThread) { + atomic_add(&fCore->fStarvationCounter, 1); + } + + if (sharedThread->fWentSleepCount == 0) + fCore->fThreadList.Remove(sharedThread); + + atomic_add(&fCore->fThreadCount, -1); + return sharedThread; + } + + pinnedThread->fEnqueued = false; + fRunQueue.Remove(pinnedThread); + return pinnedThread; +} + + +void +CPUEntry::TrackActivity(ThreadData* oldThreadData, ThreadData* nextThreadData) +{ + cpu_ent* cpuEntry = &gCPU[fCPUNumber]; + + Thread* oldThread = oldThreadData->GetThread(); + if (!thread_is_idle_thread(oldThread)) { + bigtime_t active + = (oldThread->kernel_time - cpuEntry->last_kernel_time) + + (oldThread->user_time - cpuEntry->last_user_time); + + atomic_add64(&cpuEntry->active_time, active); + oldThreadData->UpdateActivity(active); + } + + oldThreadData->ComputeLoad(); + nextThreadData->ComputeLoad(); + if (!gSingleCore && !cpuEntry->disabled) + ComputeLoad(); + + Thread* nextThread = nextThreadData->GetThread(); + if (!thread_is_idle_thread(nextThread)) { + cpuEntry->last_kernel_time = nextThread->kernel_time; + cpuEntry->last_user_time = nextThread->user_time; + + nextThreadData->fLastInterruptTime = cpuEntry->interrupt_time; + + _RequestPerformanceLevel(nextThreadData); + } +} + + +inline void +CPUEntry::_RequestPerformanceLevel(ThreadData* threadData) +{ + int32 load = std::max(threadData->GetLoad(), fCore->GetLoad()); + load = std::min(std::max(load, int32(0)), kMaxLoad); + + if (load < kTargetLoad) { + int32 delta = kTargetLoad - load; + + delta *= kTargetLoad; + delta /= kCPUPerformanceScaleMax; + + decrease_cpu_performance(delta); + } else { + bool allowBoost = !gCurrentMode->avoid_boost; + + int32 delta = load - kTargetLoad; + delta *= kMaxLoad - kTargetLoad; + delta /= kCPUPerformanceScaleMax; + + increase_cpu_performance(delta, allowBoost); + } +} + + +CPUPriorityHeap::CPUPriorityHeap(int32 cpuCount) + : + MinMaxHeap(cpuCount) +{ +} + + +void +CPUPriorityHeap::Dump() +{ + kprintf("cpu priority load\n"); + CPUEntry* entry = PeekMinimum(); + while (entry) { + int32 cpu = entry->fCPUNumber; + int32 key = GetKey(entry); + kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key, + entry->fLoad / 10); + + RemoveMinimum(); + sDebugCPUHeap.Insert(entry, key); + + entry = PeekMinimum(); + } + + entry = sDebugCPUHeap.PeekMinimum(); + while (entry) { + int32 key = GetKey(entry); + sDebugCPUHeap.RemoveMinimum(); + Insert(entry, key); + entry = sDebugCPUHeap.PeekMinimum(); + } +} + + +CoreEntry::CoreEntry() + : + fCPUCount(0), + fStarvationCounter(0), + fThreadCount(0), + fActiveTime(0), + fLoad(0), + fHighLoad(false) +{ + B_INITIALIZE_SPINLOCK(&fCPULock); + B_INITIALIZE_SPINLOCK(&fQueueLock); +} + + +void +CoreEntry::UpdateLoad() +{ + ASSERT(!gSingleCore); + + if (fCPUCount == 0) { + fLoad = 0; + return; + } + + WriteSpinLocker coreLocker(gCoreHeapsLock); + + int32 newKey = GetLoad(); + int32 oldKey = CoreLoadHeap::GetKey(this); + + ASSERT(oldKey >= 0 && oldKey <= kMaxLoad); + ASSERT(newKey >= 0 && newKey <= kMaxLoad); + + if (oldKey == newKey) + return; + + if (newKey > kHighLoad) { + if (!fHighLoad) { + gCoreLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == this); + gCoreLoadHeap.RemoveMinimum(); + + gCoreHighLoadHeap.Insert(this, newKey); + + fHighLoad = true; + } else + gCoreHighLoadHeap.ModifyKey(this, newKey); + } else if (newKey < kMediumLoad) { + if (fHighLoad) { + gCoreHighLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); + gCoreHighLoadHeap.RemoveMinimum(); + + gCoreLoadHeap.Insert(this, newKey); + + fHighLoad = false; + } else + gCoreLoadHeap.ModifyKey(this, newKey); + } else { + if (fHighLoad) + gCoreHighLoadHeap.ModifyKey(this, newKey); + else + gCoreLoadHeap.ModifyKey(this, newKey); + } +} + + +CoreLoadHeap::CoreLoadHeap(int32 coreCount) + : + MinMaxHeap(coreCount) +{ +} + + +void +CoreLoadHeap::Dump() +{ + CoreEntry* entry = PeekMinimum(); + while (entry) { + int32 key = GetKey(entry); + kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, + entry->GetLoad() / 10); + + RemoveMinimum(); + sDebugCoreHeap.Insert(entry, key); + + entry = PeekMinimum(); + } + + entry = sDebugCoreHeap.PeekMinimum(); + while (entry) { + int32 key = GetKey(entry); + sDebugCoreHeap.RemoveMinimum(); + Insert(entry, key); + entry = sDebugCoreHeap.PeekMinimum(); + } +} + + +PackageEntry::PackageEntry() + : + fIdleCoreCount(0), + fCoreCount(0) +{ + B_INITIALIZE_RW_SPINLOCK(&fCoreLock); +} + + +static int +dump_run_queue(int argc, char **argv) +{ + int32 cpuCount = smp_get_num_cpus(); + int32 coreCount = gCoreCount; + + + for (int32 i = 0; i < coreCount; i++) { + kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); + gCoreEntries[i].fRunQueue.Dump(); + } + + for (int32 i = 0; i < cpuCount; i++) { + CPUEntry* cpu = &gCPUEntries[i]; + ThreadRunQueue::ConstIterator iterator + = cpu->fRunQueue.GetConstIterator(); + + if (iterator.HasNext() + && !thread_is_idle_thread(iterator.Next()->GetThread())) { + kprintf("\nCPU %" B_PRId32 " run queue:\n", i); + cpu->fRunQueue.Dump(); + } + } + + return 0; +} + + +static int +dump_cpu_heap(int argc, char** argv) +{ + kprintf("core load\n"); + gCoreLoadHeap.Dump(); + kprintf("\n"); + gCoreHighLoadHeap.Dump(); + + for (int32 i = 0; i < gCoreCount; i++) { + if (gCoreEntries[i].fCPUCount < 2) + continue; + + kprintf("\nCore %" B_PRId32 " heap:\n", i); + gCoreEntries[i].fCPUHeap.Dump(); + } + + return 0; +} + + +static int +dump_idle_cores(int argc, char** argv) +{ + kprintf("Idle packages:\n"); + IdlePackageList::ReverseIterator idleIterator + = gIdlePackageList.GetReverseIterator(); + + if (idleIterator.HasNext()) { + kprintf("package cores\n"); + + while (idleIterator.HasNext()) { + PackageEntry* entry = idleIterator.Next(); + kprintf("%-7" B_PRId32 " ", entry->fPackageID); + + DoublyLinkedList::ReverseIterator iterator + = entry->fIdleCores.GetReverseIterator(); + if (iterator.HasNext()) { + while (iterator.HasNext()) { + CoreEntry* coreEntry = iterator.Next(); + kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + iterator.HasNext() ? ", " : ""); + } + } else + kprintf("-"); + kprintf("\n"); + } + } else + kprintf("No idle packages.\n"); + + return 0; +} + + +void Scheduler::init_debug_commands(void) +{ + new(&sDebugCPUHeap) CPUPriorityHeap(smp_get_num_cpus()); + new(&sDebugCoreHeap) CoreLoadHeap(smp_get_num_cpus()); + + add_debugger_command_etc("run_queue", &dump_run_queue, + "List threads in run queue", "\nLists threads in run queue", 0); + if (!gSingleCore) { + add_debugger_command_etc("cpu_heap", &dump_cpu_heap, + "List CPUs in CPU priority heap", + "\nList CPUs in CPU priority heap", 0); + add_debugger_command_etc("idle_cores", &dump_idle_cores, + "List idle cores", "\nList idle cores", 0); + } +} + diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h new file mode 100644 index 0000000000..14c1a80836 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -0,0 +1,170 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_CPU_H +#define KERNEL_SCHEDULER_CPU_H + + +#include + +#include +#include + +#include + +#include "RunQueue.h" +#include "scheduler_common.h" +#include "scheduler_modes.h" + + +namespace Scheduler { + + +struct ThreadData; + +struct CPUEntry; +struct CoreEntry; +struct PackageEntry; + +// The run queues. Holds the threads ready to run ordered by priority. +// One queue per schedulable target per core. Additionally, each +// logical processor has its sPinnedRunQueues used for scheduling +// pinned threads. +class ThreadRunQueue : public RunQueue { +public: + void Dump() const; +}; + +struct CPUEntry : public MinMaxHeapLinkImpl { + CPUEntry(); + + void UpdatePriority(int32 priority); + + void ComputeLoad(); + + ThreadData* ChooseNextThread(ThreadData* oldThread, + bool putAtBack); + + void TrackActivity(ThreadData* oldThreadData, + ThreadData* nextThreadData); + + int32 fCPUNumber; + CoreEntry* fCore; + + rw_spinlock fSchedulerModeLock; + + int32 fPriority; + ThreadRunQueue fRunQueue; + + int32 fLoad; + + bigtime_t fMeasureActiveTime; + bigtime_t fMeasureTime; + +private: + inline void _RequestPerformanceLevel( + ThreadData* threadData); + +} CACHE_LINE_ALIGN; + +class CPUPriorityHeap : public MinMaxHeap { +public: + CPUPriorityHeap() { } + CPUPriorityHeap(int32 cpuCount); + + void Dump(); +}; + +struct CoreEntry : public MinMaxHeapLinkImpl, + DoublyLinkedListLinkImpl { + CoreEntry(); + + inline int32 GetLoad() const; + void UpdateLoad(); + + static inline CoreEntry* GetCore(int32 cpu); + + int32 fCoreID; + PackageEntry* fPackage; + + int32 fCPUCount; + CPUPriorityHeap fCPUHeap; + spinlock fCPULock; + + int32 fStarvationCounter; + DoublyLinkedList fThreadList; + + int32 fThreadCount; + ThreadRunQueue fRunQueue; + spinlock fQueueLock; + + bigtime_t fActiveTime; + + int32 fLoad; + bool fHighLoad; +} CACHE_LINE_ALIGN; + +class CoreLoadHeap : public MinMaxHeap { +public: + CoreLoadHeap() { } + CoreLoadHeap(int32 coreCount); + + void Dump(); +}; + +// gPackageEntries are used to decide which core should be woken up from the +// idle state. When aiming for performance we should use as many packages as +// possible with as little cores active in each package as possible (so that the +// package can enter any boost mode if it has one and the active core have more +// of the shared cache for themselves. If power saving is the main priority we +// should keep active cores on as little packages as possible (so that other +// packages can go to the deep state of sleep). The heap stores only packages +// with at least one core active and one core idle. The packages with all cores +// idle are stored in gPackageIdleList (in LIFO manner). +struct PackageEntry : public DoublyLinkedListLinkImpl { + PackageEntry(); + + int32 fPackageID; + + DoublyLinkedList fIdleCores; + int32 fIdleCoreCount; + int32 fCoreCount; + rw_spinlock fCoreLock; +} CACHE_LINE_ALIGN; +typedef DoublyLinkedList IdlePackageList; + +extern CPUEntry* gCPUEntries; + +extern CoreEntry* gCoreEntries; +extern CoreLoadHeap gCoreLoadHeap; +extern CoreLoadHeap gCoreHighLoadHeap; +extern rw_spinlock gCoreHeapsLock; +extern int32 gCoreCount; + +extern PackageEntry* gPackageEntries; +extern IdlePackageList gIdlePackageList; +extern rw_spinlock gIdlePackageLock; +extern int32 gPackageCount; + + +inline int32 +CoreEntry::GetLoad() const +{ + ASSERT(fCPUCount >= 0); + return fLoad / fCPUCount; +} + + +/* static */ inline CoreEntry* +CoreEntry::GetCore(int32 cpu) +{ + return gCPUEntries[cpu].fCore; +} + + +} // namespace Scheduler + + +#endif // KERNEL_SCHEDULER_CPU_H + diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 2d790a3967..378befc2fa 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -11,26 +11,40 @@ struct scheduler_mode_operations { - const char* name; + const char* name; - bool avoid_boost; + bool avoid_boost; - bigtime_t base_quantum; - bigtime_t minimal_quantum; - bigtime_t quantum_multipliers[2]; + bigtime_t base_quantum; + bigtime_t minimal_quantum; + bigtime_t quantum_multipliers[2]; - bigtime_t maximum_latency; + bigtime_t maximum_latency; - void (*switch_to_mode)(void); - void (*set_cpu_enabled)(int32 cpu, bool enabled); - bool (*has_cache_expired)(Thread* thread); - int32 (*choose_core)(Thread* thread); - bool (*should_rebalance)(Thread* thread); - void (*rebalance_irqs)(bool idle); + void (*switch_to_mode)(void); + void (*set_cpu_enabled)(int32 cpu, bool enabled); + bool (*has_cache_expired)( + const Scheduler::ThreadData* threadData); + Scheduler::CoreEntry* (*choose_core)( + const Scheduler::ThreadData* threadData); + bool (*should_rebalance)( + const Scheduler::ThreadData* threadData); + void (*rebalance_irqs)(bool idle); }; extern struct scheduler_mode_operations gSchedulerLowLatencyMode; extern struct scheduler_mode_operations gSchedulerPowerSavingMode; + +namespace Scheduler { + + +extern scheduler_mode gCurrentModeID; +extern scheduler_mode_operations* gCurrentMode; + + +} + + #endif // KERNEL_SCHEDULER_MODES_H diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp new file mode 100644 index 0000000000..cf53fcda9f --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -0,0 +1,158 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ + +#include "scheduler_thread.h" + + +using namespace Scheduler; + + +ThreadData::ThreadData(Thread* thread) + : + fThread(thread) +{ + Init(); +} + + +void +ThreadData::Init() +{ + fPriorityPenalty = 0; + fAdditionalPenalty = 0; + + fTimeLeft = 0; + fStolenTime = 0; + + fMeasureActiveTime = 0; + fMeasureTime = 0; + fLoad = 0; + + fWentSleep = 0; + fWentSleepActive = 0; + fWentSleepCount = -1; + + fEnqueued = false; + + fCore = NULL; +} + + +void +ThreadData::Init(CoreEntry* core) +{ + Init(); + fCore = core; +} + + +void +ThreadData::Dump() const +{ + kprintf("\tpriority_penalty:\t%" B_PRId32 "\n", fPriorityPenalty); + + int32 additionalPenalty = 0; + const int kMinimalPriority = _GetMinimalPriority(); + if (kMinimalPriority > 0) + additionalPenalty = fAdditionalPenalty % kMinimalPriority; + kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", + additionalPenalty, fAdditionalPenalty); + kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", fStolenTime); + kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", fLoad / 10); + kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", fWentSleep); + kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); + kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount); + kprintf("\tcore:\t\t\t%" B_PRId32 "\n", + fCore != NULL ? fCore->fCoreID : -1); + if (fCore != NULL && HasCacheExpired()) + kprintf("\tcache affinity has expired\n"); +} + + +bool +ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU) +{ + bool rescheduleNeeded = false; + + if (targetCore == NULL && targetCPU != NULL) + targetCore = targetCPU->fCore; + else if (targetCore != NULL && targetCPU == NULL) + targetCPU = _ChooseCPU(targetCore, rescheduleNeeded); + else if (targetCore == NULL && targetCPU == NULL) { + targetCore = _ChooseCore(); + targetCPU = _ChooseCPU(targetCore, rescheduleNeeded); + } + + ASSERT(targetCore != NULL); + ASSERT(targetCPU != NULL); + + fCore = targetCore; + return rescheduleNeeded; +} + + +bigtime_t +ThreadData::ComputeQuantum() +{ + bigtime_t quantum; + if (fTimeLeft != 0) + quantum = fTimeLeft; + else + quantum = _GetBaseQuantum(); + + if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return quantum; + + quantum += fStolenTime; + fStolenTime = 0; + + int32 threadCount = (fCore->fThreadCount + 1) / fCore->fCPUCount; + threadCount = max_c(threadCount, 1); + + quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum); + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + + fTimeLeft = quantum; + fQuantumStart = system_time(); + + return quantum; +} + + +inline bigtime_t +ThreadData::_GetBaseQuantum() const +{ + int32 priority = GetEffectivePriority(); + + const bigtime_t kQuantum0 = gCurrentMode->base_quantum; + if (priority >= B_URGENT_DISPLAY_PRIORITY) + return kQuantum0; + + const bigtime_t kQuantum1 + = kQuantum0 * gCurrentMode->quantum_multipliers[0]; + if (priority > B_NORMAL_PRIORITY) { + return _ScaleQuantum(kQuantum1, kQuantum0, B_URGENT_DISPLAY_PRIORITY, + B_NORMAL_PRIORITY, priority); + } + + const bigtime_t kQuantum2 + = kQuantum0 * gCurrentMode->quantum_multipliers[1]; + return _ScaleQuantum(kQuantum2, kQuantum1, B_NORMAL_PRIORITY, + B_IDLE_PRIORITY, priority); +} + + +/* static */ bigtime_t +ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, + int32 maxPriority, int32 minPriority, int32 priority) +{ + ASSERT(priority <= maxPriority); + ASSERT(priority >= minPriority); + + bigtime_t result = (maxQuantum - minQuantum) * (priority - minPriority); + result /= maxPriority - minPriority; + return maxQuantum - result; +} + diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h new file mode 100644 index 0000000000..ffdae08035 --- /dev/null +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -0,0 +1,360 @@ +/* + * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org. + * Distributed under the terms of the MIT License. + */ +#ifndef KERNEL_SCHEDULER_THREAD_H +#define KERNEL_SCHEDULER_THREAD_H + + +#include +#include + +#include "scheduler_common.h" +#include "scheduler_cpu.h" + + +namespace Scheduler { + + +struct ThreadData : public DoublyLinkedListLinkImpl, + RunQueueLinkImpl { +public: + ThreadData(Thread* thread); + + void Init(); + void Init(CoreEntry* core); + + void Dump() const; + + inline bool HasCacheExpired() const; + inline bool ShouldRebalance() const; + + inline int32 GetEffectivePriority() const; + + inline void IncreasePenalty(); + inline void CancelPenalty(); + inline bool ShouldCancelPenalty() const; + + bool ChooseCoreAndCPU(CoreEntry*& targetCore, + CPUEntry*& targetCPU); + + inline void GoesAway(); + + inline void PutBack(); + inline void Enqueue(); + inline bool Dequeue(); + + inline void UpdateActivity(bigtime_t active); + inline void ComputeLoad(); + + inline bool HasQuantumEnded(bool wasPreempted, bool hasYielded); + bigtime_t ComputeQuantum(); + + inline Thread* GetThread() const { return fThread; } + inline int32 GetLoad() const { return fLoad; } + + inline CoreEntry* GetCore() const { return fCore; } + inline void UnassignCore() { fCore = NULL; } + + bigtime_t fStolenTime; + bigtime_t fQuantumStart; + bigtime_t fLastInterruptTime; + + bigtime_t fWentSleep; + bigtime_t fWentSleepActive; + int32 fWentSleepCount; + + bool fEnqueued; + +private: + inline int32 _GetPenalty() const; + inline int32 _GetMinimalPriority() const; + + inline CoreEntry* _ChooseCore() const; + inline CPUEntry* _ChooseCPU(CoreEntry* core, + bool& rescheduleNeeded) const; + + inline bigtime_t _GetBaseQuantum() const; + static bigtime_t _ScaleQuantum(bigtime_t maxQuantum, + bigtime_t minQuantum, int32 maxPriority, + int32 minPriority, int32 priority); + + Thread* fThread; + + int32 fPriorityPenalty; + int32 fAdditionalPenalty; + + bigtime_t fTimeLeft; + + bigtime_t fMeasureActiveTime; + bigtime_t fMeasureTime; + int32 fLoad; + + CoreEntry* fCore; +}; + + +inline bool +ThreadData::HasCacheExpired() const +{ + return gCurrentMode->has_cache_expired(this); +} + + +inline bool +ThreadData::ShouldRebalance() const +{ + ASSERT(!gSingleCore); + return gCurrentMode->should_rebalance(this); +} + + +inline int32 +ThreadData::GetEffectivePriority() const +{ + if (thread_is_idle_thread(fThread)) + return B_IDLE_PRIORITY; + if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return fThread->priority; + + int32 effectivePriority = fThread->priority; + effectivePriority -= _GetPenalty(); + + ASSERT(effectivePriority < B_FIRST_REAL_TIME_PRIORITY); + ASSERT(effectivePriority >= B_LOWEST_ACTIVE_PRIORITY); + + return effectivePriority; +} + + +inline void +ThreadData::IncreasePenalty() +{ + if (fThread->priority < B_LOWEST_ACTIVE_PRIORITY) + return; + if (fThread->priority >= B_FIRST_REAL_TIME_PRIORITY) + return; + + TRACE("increasing thread %ld penalty\n", fThread->id); + + int32 oldPenalty = fPriorityPenalty++; + + ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); + + const int kMinimalPriority = _GetMinimalPriority(); + if (fThread->priority - oldPenalty <= kMinimalPriority) { + fPriorityPenalty = oldPenalty; + fAdditionalPenalty++; + } +} + + +inline void +ThreadData::CancelPenalty() +{ + if (fPriorityPenalty != 0) + TRACE("cancelling thread %ld penalty\n", fThread->id); + + fAdditionalPenalty = 0; + fPriorityPenalty = 0; +} + + +inline bool +ThreadData::ShouldCancelPenalty() const +{ + if (fCore == NULL) + return false; + + return atomic_get(&fCore->fStarvationCounter) != fWentSleepCount + && system_time() - fWentSleep > gCurrentMode->base_quantum; +} + + +inline void +ThreadData::GoesAway() +{ + fLastInterruptTime = 0; + + fWentSleep = system_time(); + fWentSleepActive = atomic_get64(&fCore->fActiveTime); + fWentSleepCount = atomic_get(&fCore->fStarvationCounter); +} + + +inline void +ThreadData::PutBack() +{ + ComputeLoad(); + fWentSleepCount = -1; + + int32 priority = GetEffectivePriority(); + + SpinLocker runQueueLocker(fCore->fQueueLock); + ASSERT(!fEnqueued); + fEnqueued = true; + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->cpu != NULL); + + CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num]; + cpu->fRunQueue.PushFront(this, priority); + } else { + fCore->fRunQueue.PushFront(this, priority); + atomic_add(&fCore->fThreadCount, 1); + } +} + + +inline void +ThreadData::Enqueue() +{ + fThread->state = B_THREAD_READY; + ComputeLoad(); + fWentSleepCount = 0; + + int32 priority = GetEffectivePriority(); + + SpinLocker runQueueLocker(fCore->fQueueLock); + ASSERT(!fEnqueued); + fEnqueued = true; + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->previous_cpu != NULL); + + CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; + cpu->fRunQueue.PushBack(this, priority); + } else { + fCore->fRunQueue.PushBack(this, priority); + fCore->fThreadList.Insert(this); + + atomic_add(&fCore->fThreadCount, 1); + } +} + + +inline bool +ThreadData::Dequeue() +{ + SpinLocker runQueueLocker(fCore->fQueueLock); + if (!fEnqueued) + return false; + + fEnqueued = false; + if (fThread->pinned_to_cpu > 0) { + ASSERT(fThread->previous_cpu != NULL); + + CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; + cpu->fRunQueue.Remove(this); + } else { + fCore->fRunQueue.Remove(this); + + ASSERT(fWentSleepCount < 1); + if (fWentSleepCount == 0) + fCore->fThreadList.Remove(this); + atomic_add(&fCore->fThreadCount, -1); + } + + return true; +} + + +inline void +ThreadData::UpdateActivity(bigtime_t active) +{ + fMeasureActiveTime += active; + gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; + atomic_add64(&fCore->fActiveTime, active); +} + + +inline void +ThreadData::ComputeLoad() +{ + if (fLastInterruptTime > 0) { + bigtime_t interruptTime = gCPU[smp_get_current_cpu()].interrupt_time; + interruptTime -= fLastInterruptTime; + fMeasureActiveTime -= interruptTime; + } + + compute_load(fMeasureTime, fMeasureActiveTime, fLoad); +} + + +inline bool +ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) +{ + if (hasYielded) { + fTimeLeft = 0; + return true; + } + + bigtime_t timeUsed = system_time() - fQuantumStart; + fTimeLeft -= timeUsed; + fTimeLeft = std::max(fTimeLeft, bigtime_t(0)); + + // too little time left, it's better make the next quantum a bit longer + if (wasPreempted || fTimeLeft <= gCurrentMode->minimal_quantum) { + fStolenTime += fTimeLeft; + fTimeLeft = 0; + } + + return fTimeLeft == 0; +} + + +inline int32 +ThreadData::_GetPenalty() const +{ + int32 penalty = fPriorityPenalty; + + const int kMinimalPriority = _GetMinimalPriority(); + if (kMinimalPriority > 0) + penalty += fAdditionalPenalty % kMinimalPriority; + + return penalty; +} + + +inline int32 +ThreadData::_GetMinimalPriority() const +{ + const int32 kDivisor = 5; + + const int32 kMaximalPriority = 25; + const int32 kMinimalPriority = B_LOWEST_ACTIVE_PRIORITY; + + int32 priority = fThread->priority / kDivisor; + return std::max(std::min(priority, kMaximalPriority), kMinimalPriority); +} + + +inline CoreEntry* +ThreadData::_ChooseCore() const +{ + ASSERT(!gSingleCore); + return gCurrentMode->choose_core(this); +} + + +inline CPUEntry* +ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const +{ + SpinLocker cpuLocker(core->fCPULock); + CPUEntry* cpu = core->fCPUHeap.PeekMinimum(); + ASSERT(cpu != NULL); + + int32 threadPriority = GetEffectivePriority(); + if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { + cpu->UpdatePriority(threadPriority); + rescheduleNeeded = true; + } else + rescheduleNeeded = false; + + return cpu; +} + + +} // namespace Scheduler + + +#endif // KERNEL_SCHEDULER_THREAD_H +