diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 09c6f719e8..51381c3ac8 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -160,13 +160,11 @@ rebalance_irqs(bool idle) scheduler_mode_operations gSchedulerLowLatencyMode = { "low latency", - true, - 2000, - 700, - { 2, 30 }, + 100, + { 2, 25 }, - 60000, + 50000, switch_to_mode, set_cpu_enabled, diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index b41bbe7007..66efc8ce9b 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -236,8 +236,6 @@ rebalance_irqs(bool idle) scheduler_mode_operations gSchedulerPowerSavingMode = { "power saving", - false, - 3000, 1000, { 3, 60 }, diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index ca04585c92..0ee4cd0bd8 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -404,8 +404,6 @@ reschedule(int32 nextState) if (oldThreadData->HasQuantumEnded(oldThread->cpu->preempted, oldThread->has_yielded)) { - oldThreadData->IncreasePenalty(); - TRACE("enqueueing thread %ld into run queue priority = %ld\n", oldThread->id, oldThreadData->GetEffectivePriority()); putOldThreadAtBack = true; @@ -419,7 +417,6 @@ reschedule(int32 nextState) case THREAD_STATE_FREE_ON_RESCHED: break; default: - oldThreadData->IncreasePenalty(); oldThreadData->GoesAway(); TRACE("not enqueueing thread %ld into run queue next_state = %ld\n", oldThread->id, nextState); diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index 9f9eb48cef..4bdb566840 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -21,7 +21,6 @@ public: static void DumpCPURunQueue(CPUEntry* cpu); static void DumpCoreRunQueue(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); - }; @@ -43,7 +42,8 @@ ThreadRunQueue::Dump() const kprintf("%p %-7" B_PRId32 " %-8" B_PRId32 " %-8" B_PRId32 " %s\n", thread, thread->id, thread->priority, - threadData->GetEffectivePriority(), thread->name); + thread->priority - threadData->GetEffectivePriority(), + thread->name); } } } @@ -159,9 +159,6 @@ CPUEntry::UpdatePriority(int32 priority) return; fCore->CPUHeap()->ModifyKey(this, priority); - if (gSingleCore) - return; - if (oldPriority == B_IDLE_PRIORITY) fCore->CPUWakesUp(this); else if (priority == B_IDLE_PRIORITY) @@ -345,6 +342,7 @@ CoreEntry::CoreEntry() fCPUCount(0), fCPUIdleCount(0), fStarvationCounter(0), + fStarvationCounterIdle(0), fThreadCount(0), fActiveTime(0), fLoad(0), @@ -380,8 +378,6 @@ CoreEntry::PushBack(ThreadData* thread, int32 priority) SCHEDULER_ENTER_FUNCTION(); fRunQueue.PushBack(thread, priority); - fThreadList.Insert(thread); - atomic_add(&fThreadCount, 1); } @@ -393,12 +389,13 @@ CoreEntry::Remove(ThreadData* thread) ASSERT(thread->IsEnqueued()); thread->SetDequeued(); - if (thread_is_idle_thread(thread->GetThread()) - || fThreadList.Head() == thread) { + + ASSERT(!thread_is_idle_thread(thread->GetThread())); + if (thread->GetEffectivePriority() == B_LOWEST_ACTIVE_PRIORITY + || thread->IsCPUBound()) { atomic_add(&fStarvationCounter, 1); } - if (thread->WentSleepCount() == 0) - fThreadList.Remove(thread); + fRunQueue.Remove(thread); atomic_add(&fThreadCount, -1); } @@ -548,8 +545,8 @@ CoreLoadHeap::Dump() CoreEntry* entry = PeekMinimum(); while (entry) { int32 key = GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(), - entry->GetLoad() / 10); + kprintf("%4" B_PRId32 " %3" B_PRId32 "%% %7" B_PRId32 "\n", entry->ID(), + entry->GetLoad() / 10, entry->ThreadCount()); RemoveMinimum(); sDebugCoreHeap.Insert(entry, key); @@ -647,7 +644,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) static int -dump_run_queue(int argc, char **argv) +dump_run_queue(int /* argc */, char** /* argv */) { int32 cpuCount = smp_get_num_cpus(); int32 coreCount = gCoreCount; @@ -665,9 +662,9 @@ dump_run_queue(int argc, char **argv) static int -dump_cpu_heap(int argc, char** argv) +dump_cpu_heap(int /* argc */, char** /* argv */) { - kprintf("core load\n"); + kprintf("core load threads\n"); gCoreLoadHeap.Dump(); kprintf("\n"); gCoreHighLoadHeap.Dump(); @@ -685,7 +682,7 @@ dump_cpu_heap(int argc, char** argv) static int -dump_idle_cores(int argc, char** argv) +dump_idle_cores(int /* argc */, char** /* argv */) { kprintf("Idle packages:\n"); IdlePackageList::ReverseIterator idleIterator diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index 1def98d4e6..95f20bc9f9 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -150,6 +150,7 @@ public: void UpdateLoad(int32 delta); inline int32 StarvationCounter() const; + inline int32 StarvationCounterIdle() const; inline void CPUGoesIdle(CPUEntry* cpu); inline void CPUWakesUp(CPUEntry* cpu); @@ -174,7 +175,7 @@ private: spinlock fCPULock; int32 fStarvationCounter; - DoublyLinkedList fThreadList; + int32 fStarvationCounterIdle; int32 fThreadCount; ThreadRunQueue fRunQueue; @@ -387,6 +388,14 @@ CoreEntry::StarvationCounter() const } +inline int32 +CoreEntry::StarvationCounterIdle() const +{ + SCHEDULER_ENTER_FUNCTION(); + return fStarvationCounterIdle; +} + + /* PackageEntry::CoreGoesIdle and PackageEntry::CoreWakesUp have to be defined before CoreEntry::CPUGoesIdle and CoreEntry::CPUWakesUp. If they weren't GCC2 wouldn't inline them as, apparently, it doesn't do enough optimization @@ -437,8 +446,13 @@ PackageEntry::CoreWakesUp(CoreEntry* core) inline void CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) { - ASSERT(fCPUIdleCount < fCPUCount); + atomic_add(&fStarvationCounter, 1); + atomic_add(&fStarvationCounterIdle, 1); + if (gSingleCore) + return; + + ASSERT(fCPUIdleCount < fCPUCount); if (++fCPUIdleCount == fCPUCount) fPackage->CoreGoesIdle(this); } @@ -447,8 +461,10 @@ CoreEntry::CPUGoesIdle(CPUEntry* /* cpu */) inline void CoreEntry::CPUWakesUp(CPUEntry* /* cpu */) { - ASSERT(fCPUIdleCount > 0); + if (gSingleCore) + return; + ASSERT(fCPUIdleCount > 0); if (fCPUIdleCount-- == fCPUCount) fPackage->CoreWakesUp(this); } diff --git a/src/system/kernel/scheduler/scheduler_modes.h b/src/system/kernel/scheduler/scheduler_modes.h index 09670dd160..820912afd9 100644 --- a/src/system/kernel/scheduler/scheduler_modes.h +++ b/src/system/kernel/scheduler/scheduler_modes.h @@ -13,8 +13,6 @@ struct scheduler_mode_operations { const char* name; - bool avoid_boost; - bigtime_t base_quantum; bigtime_t minimal_quantum; bigtime_t quantum_multipliers[2]; diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index af4b40aa1e..e7a2445293 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -19,6 +19,9 @@ ThreadData::_InitBase() fAdditionalPenalty = 0; fEffectivePriority = fThread->priority; + fReceivedPenalty = false; + fHasSlept = false; + fTimeLeft = 0; fStolenTime = 0; @@ -28,7 +31,8 @@ ThreadData::_InitBase() fWentSleep = 0; fWentSleepActive = 0; - fWentSleepCount = -1; + fWentSleepCount = 0; + fWentSleepCountIdle = 0; fEnqueued = false; } @@ -124,7 +128,25 @@ ThreadData::Dump() const additionalPenalty = fAdditionalPenalty % kMinimalPriority; kprintf("\tadditional_penalty:\t%" B_PRId32 " (%" B_PRId32 ")\n", additionalPenalty, fAdditionalPenalty); - kprintf("\tstolen_time:\t\t%" B_PRId64 "\n", fStolenTime); + kprintf("\teffective_priority:\t%" B_PRId32 "\n", GetEffectivePriority()); + + kprintf("\treceived_penalty:\t%s\n", fReceivedPenalty ? "true" : "false"); + kprintf("\thas_slept:\t\t%s\n", fHasSlept ? "true" : "false"); + + bigtime_t quantum = _GetBaseQuantum(); + if (fThread->priority < B_FIRST_REAL_TIME_PRIORITY) { + int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount(); + threadCount = max_c(threadCount, 1); + + quantum + = std::min(gCurrentMode->maximum_latency / threadCount, quantum); + quantum = std::max(quantum, gCurrentMode->minimal_quantum); + } + kprintf("\ttime_left:\t\t%" B_PRId64 " us (quantum: %" B_PRId64 " us)\n", + fTimeLeft, quantum); + + kprintf("\tstolen_time:\t\t%" B_PRId64 " us\n", fStolenTime); + kprintf("\tquantum_start:\t\t%" B_PRId64 " us\n", fQuantumStart); kprintf("\tload:\t\t\t%" B_PRId32 "%%\n", fLoad / 10); kprintf("\twent_sleep:\t\t%" B_PRId64 "\n", fWentSleep); kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 5c8d052120..fd28497f3d 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -44,10 +44,11 @@ public: inline int32 GetEffectivePriority() const; - inline void IncreasePenalty(); inline void CancelPenalty(); inline bool ShouldCancelPenalty() const; + inline bool IsCPUBound() const { return fAdditionalPenalty != 0; } + bool ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU); @@ -58,7 +59,6 @@ public: inline void GoesAway(); inline bigtime_t WentSleep() const { return fWentSleep; } inline bigtime_t WentSleepActive() const { return fWentSleepActive; } - inline bigtime_t WentSleepCount() const { return fWentSleepCount; } inline void PutBack(); inline void Enqueue(); @@ -83,6 +83,7 @@ public: static void ComputeQuantumLengths(); private: + inline void _IncreasePenalty(); inline int32 _GetPenalty() const; void _ComputeEffectivePriority() const; @@ -98,6 +99,7 @@ private: bigtime_t fWentSleep; bigtime_t fWentSleepActive; int32 fWentSleepCount; + int32 fWentSleepCountIdle; bool fEnqueued; @@ -105,6 +107,8 @@ private: int32 fPriorityPenalty; int32 fAdditionalPenalty; + bool fReceivedPenalty; + bool fHasSlept; mutable int32 fEffectivePriority; @@ -167,7 +171,7 @@ ThreadData::GetEffectivePriority() const inline void -ThreadData::IncreasePenalty() +ThreadData::_IncreasePenalty() { SCHEDULER_ENTER_FUNCTION(); @@ -178,6 +182,7 @@ ThreadData::IncreasePenalty() TRACE("increasing thread %ld penalty\n", fThread->id); + fReceivedPenalty = true; int32 oldPenalty = fPriorityPenalty++; ASSERT(fThread->priority - oldPenalty >= B_LOWEST_ACTIVE_PRIORITY); @@ -216,9 +221,16 @@ ThreadData::ShouldCancelPenalty() const if (fCore == NULL) return false; + if (system_time() - fWentSleep > gCurrentMode->minimal_quantum * 2) + return false; - return fCore->StarvationCounter() != fWentSleepCount - && system_time() - fWentSleep > gCurrentMode->base_quantum; + if (GetEffectivePriority() != B_LOWEST_ACTIVE_PRIORITY + && !IsCPUBound()) { + if (fCore->StarvationCounter() != fWentSleepCount) + return true; + } + + return fCore->StarvationCounterIdle() != fWentSleepCountIdle; } @@ -238,10 +250,15 @@ ThreadData::GoesAway() { SCHEDULER_ENTER_FUNCTION(); + if (!fReceivedPenalty) + _IncreasePenalty(); + fHasSlept = true; + fLastInterruptTime = 0; fWentSleep = system_time(); fWentSleepCount = fCore->StarvationCounter(); + fWentSleepCountIdle = fCore->StarvationCounterIdle(); fWentSleepActive = fCore->GetActiveTime(); } @@ -253,7 +270,6 @@ ThreadData::PutBack() if (gTrackLoad) ComputeLoad(); - fWentSleepCount = -1; int32 priority = GetEffectivePriority(); @@ -285,7 +301,6 @@ ThreadData::Enqueue() if (gTrackLoad) ComputeLoad(); - fWentSleepCount = 0; int32 priority = GetEffectivePriority(); @@ -328,7 +343,7 @@ ThreadData::Dequeue() CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; - ASSERT(fWentSleepCount < 1); + fCore->Remove(this); ASSERT(!fEnqueued); return true; @@ -354,15 +369,25 @@ ThreadData::HasQuantumEnded(bool wasPreempted, bool hasYielded) } bigtime_t timeUsed = system_time() - fQuantumStart; - fTimeLeft -= timeUsed; + if (timeUsed > 0); + fTimeLeft -= timeUsed; fTimeLeft = std::max(fTimeLeft, bigtime_t(0)); // too little time left, it's better make the next quantum a bit longer - if (wasPreempted || fTimeLeft <= gCurrentMode->minimal_quantum) { + int32 skipTime = gCurrentMode->minimal_quantum; + skipTime -= skipTime / 10; + if (wasPreempted || fTimeLeft <= skipTime) { fStolenTime += fTimeLeft; fTimeLeft = 0; } + if (fTimeLeft == 0) { + if (!fReceivedPenalty && !fHasSlept) + _IncreasePenalty(); + fReceivedPenalty = false; + fHasSlept = false; + } + return fTimeLeft == 0; }