diff --git a/src/system/kernel/scheduler/low_latency.cpp b/src/system/kernel/scheduler/low_latency.cpp index 441a609a7d..74cddd6801 100644 --- a/src/system/kernel/scheduler/low_latency.cpp +++ b/src/system/kernel/scheduler/low_latency.cpp @@ -37,13 +37,7 @@ has_cache_expired(const ThreadData* threadData) CoreEntry* core = threadData->Core(); - bigtime_t activeTime; - uint32 count; - do { - count = acquire_read_seqlock(&core->fActiveTimeLock); - activeTime = core->fActiveTime; - } while (!release_read_seqlock(&core->fActiveTimeLock, count)); - + bigtime_t activeTime = core->GetActiveTime(); return activeTime - threadData->fWentSleepActive > kCacheExpire; } @@ -138,9 +132,7 @@ rebalance_irqs(bool idle) other = gCoreHighLoadHeap.PeekMinimum(); coreLocker.Unlock(); - SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; - cpuLocker.Unlock(); + int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; ASSERT(other != NULL); diff --git a/src/system/kernel/scheduler/power_saving.cpp b/src/system/kernel/scheduler/power_saving.cpp index 2ca77db7ec..d6f3a7d5bf 100644 --- a/src/system/kernel/scheduler/power_saving.cpp +++ b/src/system/kernel/scheduler/power_saving.cpp @@ -164,9 +164,7 @@ pack_irqs() irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs); locker.Unlock(); - ReadSpinLocker coreLocker(gCoreHeapsLock); - int32 newCPU = smallTaskCore->fCPUHeap.PeekMinimum()->fCPUNumber; - coreLocker.Unlock(); + int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber; if (newCPU != cpu->cpu_num) assign_io_interrupt_to_cpu(irq->irq, newCPU); @@ -209,9 +207,7 @@ rebalance_irqs(bool idle) coreLocker.Unlock(); if (other == NULL) return; - SpinLocker cpuLocker(other->fCPULock); - int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber; - cpuLocker.Unlock(); + int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber; CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu()); if (other == core) diff --git a/src/system/kernel/scheduler/scheduler.cpp b/src/system/kernel/scheduler/scheduler.cpp index 0dd94b9f0d..97e2e5256d 100644 --- a/src/system/kernel/scheduler/scheduler.cpp +++ b/src/system/kernel/scheduler/scheduler.cpp @@ -92,6 +92,11 @@ private: int fState; }; +class ThreadEnqueuer : public ThreadProcessing { +public: + void operator()(ThreadData* thread); +}; + scheduler_mode gCurrentModeID; scheduler_mode_operations* gCurrentMode; @@ -133,6 +138,16 @@ static int32* sCPUToCore; static int32* sCPUToPackage; +static void enqueue(Thread* thread, bool newOne); + + +void +ThreadEnqueuer::operator()(ThreadData* thread) +{ + enqueue(thread->GetThread(), false); +} + + void scheduler_dump_thread_data(Thread* thread) { @@ -245,7 +260,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority) ASSERT(thread->cpu != NULL); CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num]; - SpinLocker coreLocker(threadData->Core()->fCPULock); + CoreCPUHeapLocker _(threadData->Core()); cpu->UpdatePriority(priority); } @@ -467,7 +482,7 @@ reschedule(int32 nextState) ThreadData* nextThreadData; if (gCPU[thisCPU].disabled) { if (!thread_is_idle_thread(oldThread)) { - SpinLocker runQueueLocker(core->fQueueLock); + CoreRunQueueLocker _(core); nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY); cpu->fRunQueue.Remove(nextThreadData); @@ -483,7 +498,7 @@ reschedule(int32 nextState) } Thread* nextThread = nextThreadData->GetThread(); - SpinLocker cpuLocker(core->fCPULock); + CoreCPUHeapLocker cpuLocker(core); cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); cpuLocker.Unlock(); @@ -511,9 +526,9 @@ reschedule(int32 nextState) nextThread->state = B_THREAD_RUNNING; // update CPU heap - SpinLocker coreLocker(core->fCPULock); + cpuLocker.Lock(); cpu->UpdatePriority(nextThreadData->GetEffectivePriority()); - coreLocker.Unlock(); + cpuLocker.Unlock(); // track kernel time (user time is tracked in thread_at_kernel_entry()) update_thread_times(oldThread, nextThread); @@ -631,18 +646,6 @@ scheduler_set_operation_mode(scheduler_mode mode) } -static void -unassign_thread(Thread* thread, void* data) -{ - CoreEntry* core = static_cast(data); - - if (thread->scheduler_data->Core() == core - && thread->pinned_to_cpu == 0) { - thread->scheduler_data->UnassignCore(); - } -} - - void scheduler_set_cpu_enabled(int32 cpuID, bool enabled) { @@ -660,78 +663,21 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled) CPUEntry* cpu = &gCPUEntries[cpuID]; CoreEntry* core = cpu->fCore; - PackageEntry* package = core->fPackage; - int32 oldCPUCount = core->fCPUCount; + int32 oldCPUCount = core->CPUCount(); ASSERT(oldCPUCount >= 0); - if (enabled) - core->fCPUCount++; - else { + if (enabled) { + cpu->fLoad = 0; + core->AddCPU(cpu); + } else { cpu->UpdatePriority(B_IDLE_PRIORITY); - core->fCPUCount--; + + ThreadEnqueuer enqueuer; + core->RemoveCPU(cpu, enqueuer); } gCPU[cpuID].disabled = !enabled; - if (core->fCPUCount == 0) { - // core has been disabled - ASSERT(!enabled); - - if (core->fHighLoad) { - gCoreHighLoadHeap.ModifyKey(core, -1); - ASSERT(gCoreHighLoadHeap.PeekMinimum() == core); - gCoreHighLoadHeap.RemoveMinimum(); - } else { - gCoreLoadHeap.ModifyKey(core, -1); - ASSERT(gCoreLoadHeap.PeekMinimum() == core); - gCoreLoadHeap.RemoveMinimum(); - } - - package->RemoveIdleCore(core); - - // get rid of threads - thread_map(unassign_thread, core); - - core->fThreadCount = 0; - while (core->fRunQueue.PeekMaximum() != NULL) { - ThreadData* threadData = core->fRunQueue.PeekMaximum(); - - core->fRunQueue.Remove(threadData); - threadData->fEnqueued = false; - - if (threadData->fWentSleepCount == 0) { - core->fThreadList.Remove(threadData); - threadData->fWentSleepCount = -1; - } - - ASSERT(threadData->Core() == NULL); - enqueue(threadData->GetThread(), false); - } - } else if (oldCPUCount == 0) { - // core has been reenabled - ASSERT(enabled); - - cpu->fLoad = 0; - core->fLoad = 0; - core->fHighLoad = false; - gCoreLoadHeap.Insert(core, 0); - - package->AddIdleCore(core); - } - - if (enabled) { - core->fCPUHeap.Insert(cpu, B_IDLE_PRIORITY); - cpu->fLoad = 0; - } else { - core->fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); - ASSERT(core->fCPUHeap.PeekMaximum() == cpu); - core->fCPUHeap.RemoveMaximum(); - - ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); - core->fLoad -= cpu->fLoad; - ASSERT(core->fLoad >= 0); - } - if (!enabled) { cpu_ent* entry = &gCPU[cpuID]; @@ -856,32 +802,17 @@ init() new(&gIdlePackageList) IdlePackageList; - for (int32 i = 0; i < packageCount; i++) - gPackageEntries[i].Init(i); - - for (int32 i = 0; i < coreCount; i++) { - gCoreEntries[i].fCoreID = i; - gCoreEntries[i].fCPUCount = cpuCount / coreCount; - - result = gCoreLoadHeap.Insert(&gCoreEntries[i], 0); - if (result != B_OK) - return result; - } - for (int32 i = 0; i < cpuCount; i++) { CoreEntry* core = &gCoreEntries[sCPUToCore[i]]; PackageEntry* package = &gPackageEntries[sCPUToPackage[i]]; + package->Init(sCPUToPackage[i]); + core->Init(sCPUToCore[i], package); + gCPUEntries[i].fCPUNumber = i; gCPUEntries[i].fCore = core; - core->fPackage = package; - if (core->fCPUHeap.PeekMaximum() == NULL) - package->AddIdleCore(core); - - result = core->fCPUHeap.Insert(&gCPUEntries[i], B_IDLE_PRIORITY); - if (result != B_OK) - return result; + core->AddCPU(&gCPUEntries[i]); } packageEntriesDeleter.Detach(); @@ -981,9 +912,9 @@ _user_estimate_max_scheduling_latency(thread_id id) if (core == NULL) core = &gCoreEntries[get_random() % gCoreCount]; - int32 threadCount = core->fThreadCount; - if (core->fCPUCount > 0) - threadCount /= core->fCPUCount; + int32 threadCount = core->ThreadCount(); + if (core->CPUCount() > 0) + threadCount /= core->CPUCount(); if (threadData->GetEffectivePriority() > 0) { threadCount -= threadCount * THREAD_MAX_SET_PRIORITY diff --git a/src/system/kernel/scheduler/scheduler_cpu.cpp b/src/system/kernel/scheduler/scheduler_cpu.cpp index f4d55ebe4d..896447e765 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.cpp +++ b/src/system/kernel/scheduler/scheduler_cpu.cpp @@ -18,6 +18,7 @@ using namespace Scheduler; class Scheduler::DebugDumper { public: + static void DumpCoreRunQueue(CoreEntry* core); static void DumpIdleCoresInPackage(PackageEntry* package); }; @@ -63,17 +64,18 @@ CPUEntry::UpdatePriority(int32 priority) if (gCPU[fCPUNumber].disabled) return; - int32 corePriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); - fCore->fCPUHeap.ModifyKey(this, priority); + CPUPriorityHeap* cpuHeap = fCore->CPUHeap(); + int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); + cpuHeap->ModifyKey(this, priority); if (gSingleCore) return; - int32 maxPriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum()); + int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum()); if (corePriority == maxPriority) return; - PackageEntry* packageEntry = fCore->fPackage; + PackageEntry* packageEntry = fCore->Package(); if (maxPriority == B_IDLE_PRIORITY) packageEntry->CoreGoesIdle(fCore); else if (corePriority == B_IDLE_PRIORITY) @@ -93,9 +95,7 @@ CPUEntry::ComputeLoad() if (oldLoad != fLoad) { int32 delta = fLoad - oldLoad; - atomic_add(&fCore->fLoad, delta); - - fCore->UpdateLoad(); + fCore->UpdateLoad(delta); } if (fLoad > kVeryHighLoad) @@ -106,9 +106,9 @@ CPUEntry::ComputeLoad() ThreadData* CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) { - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); - ThreadData* sharedThread = fCore->fRunQueue.PeekMaximum(); + ThreadData* sharedThread = fCore->PeekThread(); ThreadData* pinnedThread = fRunQueue.PeekMaximum(); ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL); @@ -132,16 +132,7 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack) if (sharedPriority > pinnedPriority) { sharedThread->fEnqueued = false; - fCore->fRunQueue.Remove(sharedThread); - if (thread_is_idle_thread(sharedThread->GetThread()) - || fCore->fThreadList.Head() == sharedThread) { - atomic_add(&fCore->fStarvationCounter, 1); - } - - if (sharedThread->fWentSleepCount == 0) - fCore->fThreadList.Remove(sharedThread); - - atomic_add(&fCore->fThreadCount, -1); + fCore->Remove(sharedThread, sharedThread->fWentSleepCount == 0); return sharedThread; } @@ -259,15 +250,62 @@ CoreEntry::CoreEntry() void -CoreEntry::UpdateLoad() +CoreEntry::Init(int32 id, PackageEntry* package) { - ASSERT(!gSingleCore); + fCoreID = id; + fPackage = package; +} + +void +CoreEntry::PushFront(ThreadData* thread, int32 priority) +{ + fRunQueue.PushFront(thread, priority); + atomic_add(&fThreadCount, 1); +} + + +void +CoreEntry::PushBack(ThreadData* thread, int32 priority) +{ + fRunQueue.PushBack(thread, priority); + fThreadList.Insert(thread); + + atomic_add(&fThreadCount, 1); +} + + +void +CoreEntry::Remove(ThreadData* thread, bool starving) +{ + if (thread_is_idle_thread(thread->GetThread()) + || fThreadList.Head() == thread) { + atomic_add(&fStarvationCounter, 1); + } + if (starving) + fThreadList.Remove(thread); + fRunQueue.Remove(thread); + atomic_add(&fThreadCount, -1); +} + + +inline ThreadData* +CoreEntry::PeekThread() const +{ + return fRunQueue.PeekMaximum(); +} + + +void +CoreEntry::UpdateLoad(int32 delta) +{ if (fCPUCount == 0) { fLoad = 0; return; } + atomic_add(&fLoad, delta); + WriteSpinLocker coreLocker(gCoreHeapsLock); int32 newKey = GetLoad(); @@ -310,6 +348,81 @@ CoreEntry::UpdateLoad() } +void +CoreEntry::AddCPU(CPUEntry* cpu) +{ + ASSERT(fCPUCount >= 0); + if (fCPUCount++ == 0) { + // core has been reenabled + fLoad = 0; + fHighLoad = false; + gCoreLoadHeap.Insert(this, 0); + + fPackage->AddIdleCore(this); + } + + fCPUHeap.Insert(cpu, B_IDLE_PRIORITY); +} + + +void +CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing) +{ + ASSERT(fCPUCount > 0); + if (--fCPUCount == 0) { + // core has been disabled + if (fHighLoad) { + gCoreHighLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreHighLoadHeap.PeekMinimum() == this); + gCoreHighLoadHeap.RemoveMinimum(); + } else { + gCoreLoadHeap.ModifyKey(this, -1); + ASSERT(gCoreLoadHeap.PeekMinimum() == this); + gCoreLoadHeap.RemoveMinimum(); + } + + fPackage->RemoveIdleCore(this); + + // get rid of threads + thread_map(CoreEntry::_UnassignThread, this); + + fThreadCount = 0; + while (fRunQueue.PeekMaximum() != NULL) { + ThreadData* threadData = fRunQueue.PeekMaximum(); + + fRunQueue.Remove(threadData); + threadData->fEnqueued = false; + + if (threadData->fWentSleepCount == 0) + fThreadList.Remove(threadData); + threadData->fWentSleepCount = -1; + + ASSERT(threadData->Core() == NULL); + threadPostProcessing(threadData); + } + } + + fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1); + ASSERT(fCPUHeap.PeekMaximum() == cpu); + fCPUHeap.RemoveMaximum(); + + ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad); + fLoad -= cpu->fLoad; + ASSERT(fLoad >= 0); +} + + +/* static */ void +CoreEntry::_UnassignThread(Thread* thread, void* data) +{ + CoreEntry* core = static_cast(data); + ThreadData* threadData = thread->scheduler_data; + + if (threadData->Core() == core && thread->pinned_to_cpu == 0) + threadData->UnassignCore(); +} + + CoreLoadHeap::CoreLoadHeap(int32 coreCount) : MinMaxHeap(coreCount) @@ -323,7 +436,7 @@ CoreLoadHeap::Dump() CoreEntry* entry = PeekMinimum(); while (entry) { int32 key = GetKey(entry); - kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID, + kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(), entry->GetLoad() / 10); RemoveMinimum(); @@ -420,6 +533,13 @@ PackageEntry::RemoveIdleCore(CoreEntry* core) } +/* static */ void +DebugDumper::DumpCoreRunQueue(CoreEntry* core) +{ + core->fRunQueue.Dump(); +} + + /* static */ void DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) { @@ -430,7 +550,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package) if (iterator.HasNext()) { while (iterator.HasNext()) { CoreEntry* coreEntry = iterator.Next(); - kprintf("%" B_PRId32 "%s", coreEntry->fCoreID, + kprintf("%" B_PRId32 "%s", coreEntry->ID(), iterator.HasNext() ? ", " : ""); } } else @@ -445,10 +565,9 @@ dump_run_queue(int argc, char **argv) int32 cpuCount = smp_get_num_cpus(); int32 coreCount = gCoreCount; - for (int32 i = 0; i < coreCount; i++) { kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i); - gCoreEntries[i].fRunQueue.Dump(); + DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]); } for (int32 i = 0; i < cpuCount; i++) { @@ -476,11 +595,11 @@ dump_cpu_heap(int argc, char** argv) gCoreHighLoadHeap.Dump(); for (int32 i = 0; i < gCoreCount; i++) { - if (gCoreEntries[i].fCPUCount < 2) + if (gCoreEntries[i].CPUCount() < 2) continue; kprintf("\nCore %" B_PRId32 " heap:\n", i); - gCoreEntries[i].fCPUHeap.Dump(); + gCoreEntries[i].CPUHeap()->Dump(); } return 0; diff --git a/src/system/kernel/scheduler/scheduler_cpu.h b/src/system/kernel/scheduler/scheduler_cpu.h index be3721352e..6df157ea13 100644 --- a/src/system/kernel/scheduler/scheduler_cpu.h +++ b/src/system/kernel/scheduler/scheduler_cpu.h @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -24,6 +25,7 @@ namespace Scheduler { class DebugDumper; struct ThreadData; +class ThreadProcessing; struct CPUEntry; struct CoreEntry; @@ -77,15 +79,57 @@ public: void Dump(); }; -struct CoreEntry : public MinMaxHeapLinkImpl, - DoublyLinkedListLinkImpl { +class CoreEntry : public MinMaxHeapLinkImpl, + public DoublyLinkedListLinkImpl { +public: CoreEntry(); + void Init(int32 id, PackageEntry* package); + + inline int32 ID() const { return fCoreID; } + inline PackageEntry* Package() const { return fPackage; } + inline int32 CPUCount() const + { return fCPUCount; } + + inline void LockCPUHeap(); + inline void UnlockCPUHeap(); + + inline CPUPriorityHeap* CPUHeap(); + + inline int32 ThreadCount() const + { return fThreadCount; } + + inline void LockRunQueue(); + inline void UnlockRunQueue(); + + void PushFront(ThreadData* thread, + int32 priority); + void PushBack(ThreadData* thread, + int32 priority); + void Remove(ThreadData* thread, + bool starving); + inline ThreadData* PeekThread() const; + + inline bigtime_t GetActiveTime() const; + inline void IncreaseActiveTime( + bigtime_t activeTime); + inline int32 GetLoad() const; - void UpdateLoad(); + void UpdateLoad(int32 delta); + + inline int32 StarvationCounter() const; + + void AddCPU(CPUEntry* cpu); + void RemoveCPU(CPUEntry* cpu, + ThreadProcessing& + threadPostProcessing); static inline CoreEntry* GetCore(int32 cpu); +private: + static void _UnassignThread(Thread* thread, + void* core); + int32 fCoreID; PackageEntry* fPackage; @@ -101,12 +145,46 @@ struct CoreEntry : public MinMaxHeapLinkImpl, spinlock fQueueLock; bigtime_t fActiveTime; - seqlock fActiveTimeLock; + mutable seqlock fActiveTimeLock; int32 fLoad; bool fHighLoad; + + friend class DebugDumper; } CACHE_LINE_ALIGN; +class CoreRunQueueLocking { +public: + inline bool Lock(CoreEntry* core) + { + core->LockRunQueue(); + return true; + } + + inline void Unlock(CoreEntry* core) + { + core->UnlockRunQueue(); + } +}; + +typedef AutoLocker CoreRunQueueLocker; + +class CoreCPUHeapLocking { +public: + inline bool Lock(CoreEntry* core) + { + core->LockCPUHeap(); + return true; + } + + inline void Unlock(CoreEntry* core) + { + core->UnlockCPUHeap(); + } +}; + +typedef AutoLocker CoreCPUHeapLocker; + class CoreLoadHeap : public MinMaxHeap { public: CoreLoadHeap() { } @@ -167,6 +245,64 @@ extern rw_spinlock gIdlePackageLock; extern int32 gPackageCount; +inline void +CoreEntry::LockCPUHeap() +{ + acquire_spinlock(&fCPULock); +} + + +inline void +CoreEntry::UnlockCPUHeap() +{ + release_spinlock(&fCPULock); +} + + +inline CPUPriorityHeap* +CoreEntry::CPUHeap() +{ + return &fCPUHeap; +} + + +inline void +CoreEntry::LockRunQueue() +{ + acquire_spinlock(&fQueueLock); +} + + +inline void +CoreEntry::UnlockRunQueue() +{ + release_spinlock(&fQueueLock); +} + + +inline void +CoreEntry::IncreaseActiveTime(bigtime_t activeTime) +{ + WriteSequentialLocker _(fActiveTimeLock); + fActiveTime += activeTime; +} + + +inline bigtime_t +CoreEntry::GetActiveTime() const +{ + bigtime_t activeTime; + + uint32 count; + do { + count = acquire_read_seqlock(&fActiveTimeLock); + activeTime = fActiveTime; + } while (!release_read_seqlock(&fActiveTimeLock, count)); + + return activeTime; +} + + inline int32 CoreEntry::GetLoad() const { @@ -175,6 +311,13 @@ CoreEntry::GetLoad() const } +inline int32 +CoreEntry::StarvationCounter() const +{ + return fStarvationCounter; +} + + /* static */ inline CoreEntry* CoreEntry::GetCore(int32 cpu) { diff --git a/src/system/kernel/scheduler/scheduler_thread.cpp b/src/system/kernel/scheduler/scheduler_thread.cpp index e1f280e1c7..5b933a5fbe 100644 --- a/src/system/kernel/scheduler/scheduler_thread.cpp +++ b/src/system/kernel/scheduler/scheduler_thread.cpp @@ -65,7 +65,7 @@ ThreadData::Dump() const kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive); kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount); kprintf("\tcore:\t\t\t%" B_PRId32 "\n", - fCore != NULL ? fCore->fCoreID : -1); + fCore != NULL ? fCore->ID() : -1); if (fCore != NULL && HasCacheExpired()) kprintf("\tcache affinity has expired\n"); } @@ -108,7 +108,7 @@ ThreadData::ComputeQuantum() quantum += fStolenTime; fStolenTime = 0; - int32 threadCount = (fCore->fThreadCount + 1) / fCore->fCPUCount; + int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount(); threadCount = max_c(threadCount, 1); quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum); @@ -137,7 +137,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const if (fThread->previous_cpu != NULL) { CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num]; if (previousCPU->fCore == core) { - SpinLocker cpuLocker(core->fCPULock); + CoreCPUHeapLocker _(core); if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) { previousCPU->UpdatePriority(threadPriority); rescheduleNeeded = true; @@ -146,8 +146,8 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const } } - SpinLocker cpuLocker(core->fCPULock); - CPUEntry* cpu = core->fCPUHeap.PeekMinimum(); + CoreCPUHeapLocker _(core); + CPUEntry* cpu = core->CPUHeap()->PeekMinimum(); ASSERT(cpu != NULL); if (CPUPriorityHeap::GetKey(cpu) < threadPriority) { @@ -195,3 +195,8 @@ ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum, return maxQuantum - result; } + +ThreadProcessing::~ThreadProcessing() +{ +} + diff --git a/src/system/kernel/scheduler/scheduler_thread.h b/src/system/kernel/scheduler/scheduler_thread.h index 7ebf59133e..2a49b68925 100644 --- a/src/system/kernel/scheduler/scheduler_thread.h +++ b/src/system/kernel/scheduler/scheduler_thread.h @@ -93,6 +93,13 @@ private: CoreEntry* fCore; }; +class ThreadProcessing { +public: + virtual ~ThreadProcessing(); + + virtual void operator()(ThreadData* thread) = 0; +}; + inline bool ThreadData::HasCacheExpired() const @@ -166,7 +173,7 @@ ThreadData::ShouldCancelPenalty() const if (fCore == NULL) return false; - return atomic_get(&fCore->fStarvationCounter) != fWentSleepCount + return fCore->StarvationCounter() != fWentSleepCount && system_time() - fWentSleep > gCurrentMode->base_quantum; } @@ -177,13 +184,8 @@ ThreadData::GoesAway() fLastInterruptTime = 0; fWentSleep = system_time(); - fWentSleepCount = atomic_get(&fCore->fStarvationCounter); - - uint32 count; - do { - count = acquire_read_seqlock(&fCore->fActiveTimeLock); - fWentSleepActive = fCore->fActiveTime; - } while (!release_read_seqlock(&fCore->fActiveTimeLock, count)); + fWentSleepCount = fCore->StarvationCounter(); + fWentSleepActive = fCore->GetActiveTime(); } @@ -195,7 +197,7 @@ ThreadData::PutBack() int32 priority = GetEffectivePriority(); - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); ASSERT(!fEnqueued); fEnqueued = true; if (fThread->pinned_to_cpu > 0) { @@ -203,10 +205,9 @@ ThreadData::PutBack() CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num]; cpu->fRunQueue.PushFront(this, priority); - } else { - fCore->fRunQueue.PushFront(this, priority); - atomic_add(&fCore->fThreadCount, 1); - } + } else + fCore->PushFront(this, priority); + fCore->UnlockRunQueue(); } @@ -219,7 +220,7 @@ ThreadData::Enqueue() int32 priority = GetEffectivePriority(); - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); ASSERT(!fEnqueued); fEnqueued = true; if (fThread->pinned_to_cpu > 0) { @@ -227,19 +228,15 @@ ThreadData::Enqueue() CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; cpu->fRunQueue.PushBack(this, priority); - } else { - fCore->fRunQueue.PushBack(this, priority); - fCore->fThreadList.Insert(this); - - atomic_add(&fCore->fThreadCount, 1); - } + } else + fCore->PushBack(this, priority); } inline bool ThreadData::Dequeue() { - SpinLocker runQueueLocker(fCore->fQueueLock); + CoreRunQueueLocker _(fCore); if (!fEnqueued) return false; @@ -250,12 +247,8 @@ ThreadData::Dequeue() CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num]; cpu->fRunQueue.Remove(this); } else { - fCore->fRunQueue.Remove(this); - ASSERT(fWentSleepCount < 1); - if (fWentSleepCount == 0) - fCore->fThreadList.Remove(this); - atomic_add(&fCore->fThreadCount, -1); + fCore->Remove(this, fWentSleepCount == 0); } return true; @@ -268,9 +261,8 @@ ThreadData::UpdateActivity(bigtime_t active) fMeasureActiveTime += active; gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active; - WriteSequentialLocker locker(fCore->fActiveTimeLock); - fCore->fActiveTime += active; - locker.Unlock(); + fCore->IncreaseActiveTime(active); + }