scheduler: Encapsulate CoreEntry fields

src/system/kernel/scheduler/low_latency.cpp

@@ -37,13 +37,7 @@ has_cache_expired(const ThreadData* threadData)
 
 	CoreEntry* core = threadData->Core();
 
-	bigtime_t activeTime;
+	bigtime_t activeTime = core->GetActiveTime();
-	uint32 count;
-	do {
-		count = acquire_read_seqlock(&core->fActiveTimeLock);
-		activeTime = core->fActiveTime;
-	} while (!release_read_seqlock(&core->fActiveTimeLock, count));
-
 	return activeTime - threadData->fWentSleepActive > kCacheExpire;
 }
 
@@ -138,9 +132,7 @@ rebalance_irqs(bool idle)
 		other = gCoreHighLoadHeap.PeekMinimum();
 	coreLocker.Unlock();
 
-	SpinLocker cpuLocker(other->fCPULock);
+	int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
-	int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber;
-	cpuLocker.Unlock();
 
 	ASSERT(other != NULL);
 

src/system/kernel/scheduler/power_saving.cpp

@@ -164,9 +164,7 @@ pack_irqs()
 		irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs);
 		locker.Unlock();
 
-		ReadSpinLocker coreLocker(gCoreHeapsLock);
+		int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber;
-		int32 newCPU = smallTaskCore->fCPUHeap.PeekMinimum()->fCPUNumber;
-		coreLocker.Unlock();
 
 		if (newCPU != cpu->cpu_num)
 			assign_io_interrupt_to_cpu(irq->irq, newCPU);
@@ -209,9 +207,7 @@ rebalance_irqs(bool idle)
 	coreLocker.Unlock();
 	if (other == NULL)
 		return;
-	SpinLocker cpuLocker(other->fCPULock);
+	int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
-	int32 newCPU = other->fCPUHeap.PeekMinimum()->fCPUNumber;
-	cpuLocker.Unlock();
 
 	CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu());
 	if (other == core)

src/system/kernel/scheduler/scheduler.cpp

@@ -92,6 +92,11 @@ private:
 	int		fState;
 };
 
+class ThreadEnqueuer : public ThreadProcessing {
+public:
+	void		operator()(ThreadData* thread);
+};
+
 scheduler_mode gCurrentModeID;
 scheduler_mode_operations* gCurrentMode;
 
@@ -133,6 +138,16 @@ static int32* sCPUToCore;
 static int32* sCPUToPackage;
 
 
+static void enqueue(Thread* thread, bool newOne);
+
+
+void
+ThreadEnqueuer::operator()(ThreadData* thread)
+{
+	enqueue(thread->GetThread(), false);
+}
+
+
 void
 scheduler_dump_thread_data(Thread* thread)
 {
@@ -245,7 +260,7 @@ scheduler_set_thread_priority(Thread *thread, int32 priority)
 			ASSERT(thread->cpu != NULL);
 			CPUEntry* cpu = &gCPUEntries[thread->cpu->cpu_num];
 
-			SpinLocker coreLocker(threadData->Core()->fCPULock);
+			CoreCPUHeapLocker _(threadData->Core());
 			cpu->UpdatePriority(priority);
 		}
 
@@ -467,7 +482,7 @@ reschedule(int32 nextState)
 	ThreadData* nextThreadData;
 	if (gCPU[thisCPU].disabled) {
 		if (!thread_is_idle_thread(oldThread)) {
-			SpinLocker runQueueLocker(core->fQueueLock);
+			CoreRunQueueLocker _(core);
 
 			nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY);
 		 	cpu->fRunQueue.Remove(nextThreadData);
@@ -483,7 +498,7 @@ reschedule(int32 nextState)
 	}
 
 	Thread* nextThread = nextThreadData->GetThread();
-	SpinLocker cpuLocker(core->fCPULock);
+	CoreCPUHeapLocker cpuLocker(core);
 	cpu->UpdatePriority(nextThreadData->GetEffectivePriority());
 	cpuLocker.Unlock();
 
@@ -511,9 +526,9 @@ reschedule(int32 nextState)
 	nextThread->state = B_THREAD_RUNNING;
 
 	// update CPU heap
-	SpinLocker coreLocker(core->fCPULock);
+	cpuLocker.Lock();
 	cpu->UpdatePriority(nextThreadData->GetEffectivePriority());
-	coreLocker.Unlock();
+	cpuLocker.Unlock();
 
 	// track kernel time (user time is tracked in thread_at_kernel_entry())
 	update_thread_times(oldThread, nextThread);
@@ -631,18 +646,6 @@ scheduler_set_operation_mode(scheduler_mode mode)
 }
 
 
-static void
-unassign_thread(Thread* thread, void* data)
-{
-	CoreEntry* core = static_cast<CoreEntry*>(data);
-
-	if (thread->scheduler_data->Core() == core
-		&& thread->pinned_to_cpu == 0) {
-		thread->scheduler_data->UnassignCore();
-	}
-}
-
-
 void
 scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
 {
@@ -660,78 +663,21 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
 
 	CPUEntry* cpu = &gCPUEntries[cpuID];
 	CoreEntry* core = cpu->fCore;
-	PackageEntry* package = core->fPackage;
 
-	int32 oldCPUCount = core->fCPUCount;
+	int32 oldCPUCount = core->CPUCount();
 	ASSERT(oldCPUCount >= 0);
-	if (enabled)
+	if (enabled) {
-		core->fCPUCount++;
+		cpu->fLoad = 0;
-	else {
+		core->AddCPU(cpu);
+	} else {
 		cpu->UpdatePriority(B_IDLE_PRIORITY);
-		core->fCPUCount--;
+
+		ThreadEnqueuer enqueuer;
+		core->RemoveCPU(cpu, enqueuer);
 	}
 
 	gCPU[cpuID].disabled = !enabled;
 
-	if (core->fCPUCount == 0) {
-		// core has been disabled
-		ASSERT(!enabled);
-
-		if (core->fHighLoad) {
-			gCoreHighLoadHeap.ModifyKey(core, -1);
-			ASSERT(gCoreHighLoadHeap.PeekMinimum() == core);
-			gCoreHighLoadHeap.RemoveMinimum();
-		} else {
-			gCoreLoadHeap.ModifyKey(core, -1);
-			ASSERT(gCoreLoadHeap.PeekMinimum() == core);
-			gCoreLoadHeap.RemoveMinimum();
-		}
-
-		package->RemoveIdleCore(core);
-
-		// get rid of threads
-		thread_map(unassign_thread, core);
-
-		core->fThreadCount = 0;
-		while (core->fRunQueue.PeekMaximum() != NULL) {
-			ThreadData* threadData = core->fRunQueue.PeekMaximum();
-
-			core->fRunQueue.Remove(threadData);
-			threadData->fEnqueued = false;
-
-			if (threadData->fWentSleepCount == 0) {
-				core->fThreadList.Remove(threadData);
-				threadData->fWentSleepCount = -1;
-			}
-
-			ASSERT(threadData->Core() == NULL);
-			enqueue(threadData->GetThread(), false);
-		}
-	} else if (oldCPUCount == 0) {
-		// core has been reenabled
-		ASSERT(enabled);
-
-		cpu->fLoad = 0;
-		core->fLoad = 0;
-		core->fHighLoad = false;
-		gCoreLoadHeap.Insert(core, 0);
-
-		package->AddIdleCore(core);
-	}
-
-	if (enabled) {
-		core->fCPUHeap.Insert(cpu, B_IDLE_PRIORITY);
-		cpu->fLoad = 0;
-	} else {
-		core->fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1);
-		ASSERT(core->fCPUHeap.PeekMaximum() == cpu);
-		core->fCPUHeap.RemoveMaximum();
-
-		ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad);
-		core->fLoad -= cpu->fLoad;
-		ASSERT(core->fLoad >= 0);
-	}
-
 	if (!enabled) {
 		cpu_ent* entry = &gCPU[cpuID];
 
@@ -856,32 +802,17 @@ init()
 
 	new(&gIdlePackageList) IdlePackageList;
 
-	for (int32 i = 0; i < packageCount; i++)
-		gPackageEntries[i].Init(i);
-
-	for (int32 i = 0; i < coreCount; i++) {
-		gCoreEntries[i].fCoreID = i;
-		gCoreEntries[i].fCPUCount = cpuCount / coreCount;
-
-		result = gCoreLoadHeap.Insert(&gCoreEntries[i], 0);
-		if (result != B_OK)
-			return result;
-	}
-
 	for (int32 i = 0; i < cpuCount; i++) {
 		CoreEntry* core = &gCoreEntries[sCPUToCore[i]];
 		PackageEntry* package = &gPackageEntries[sCPUToPackage[i]];
 
+		package->Init(sCPUToPackage[i]);
+		core->Init(sCPUToCore[i], package);
+
 		gCPUEntries[i].fCPUNumber = i;
 		gCPUEntries[i].fCore = core;
-		core->fPackage = package;
 
-		if (core->fCPUHeap.PeekMaximum() == NULL)
+		core->AddCPU(&gCPUEntries[i]);
-			package->AddIdleCore(core);
-
-		result = core->fCPUHeap.Insert(&gCPUEntries[i], B_IDLE_PRIORITY);
-		if (result != B_OK)
-			return result;
 	}
 
 	packageEntriesDeleter.Detach();
@@ -981,9 +912,9 @@ _user_estimate_max_scheduling_latency(thread_id id)
 	if (core == NULL)
 		core = &gCoreEntries[get_random<int32>() % gCoreCount];
 
-	int32 threadCount = core->fThreadCount;
+	int32 threadCount = core->ThreadCount();
-	if (core->fCPUCount > 0)
+	if (core->CPUCount() > 0)
-		threadCount /= core->fCPUCount;
+		threadCount /= core->CPUCount();
 
 	if (threadData->GetEffectivePriority() > 0) {
 		threadCount -= threadCount * THREAD_MAX_SET_PRIORITY

src/system/kernel/scheduler/scheduler_cpu.cpp

@@ -18,6 +18,7 @@ using namespace Scheduler;
 
 class Scheduler::DebugDumper {
 public:
+	static	void		DumpCoreRunQueue(CoreEntry* core);
 	static	void		DumpIdleCoresInPackage(PackageEntry* package);
 
 };
@@ -63,17 +64,18 @@ CPUEntry::UpdatePriority(int32 priority)
 	if (gCPU[fCPUNumber].disabled)
 		return;
 
-	int32 corePriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum());
+	CPUPriorityHeap* cpuHeap = fCore->CPUHeap();
-	fCore->fCPUHeap.ModifyKey(this, priority);
+	int32 corePriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum());
+	cpuHeap->ModifyKey(this, priority);
 
 	if (gSingleCore)
 		return;
 
-	int32 maxPriority = CPUPriorityHeap::GetKey(fCore->fCPUHeap.PeekMaximum());
+	int32 maxPriority = CPUPriorityHeap::GetKey(cpuHeap->PeekMaximum());
 	if (corePriority == maxPriority)
 		return;
 
-	PackageEntry* packageEntry = fCore->fPackage;
+	PackageEntry* packageEntry = fCore->Package();
 	if (maxPriority == B_IDLE_PRIORITY)
 		packageEntry->CoreGoesIdle(fCore);
 	else if (corePriority == B_IDLE_PRIORITY)
@@ -93,9 +95,7 @@ CPUEntry::ComputeLoad()
 
 	if (oldLoad != fLoad) {
 		int32 delta = fLoad - oldLoad;
-		atomic_add(&fCore->fLoad, delta);
+		fCore->UpdateLoad(delta);
-
-		fCore->UpdateLoad();
 	}
 
 	if (fLoad > kVeryHighLoad)
@@ -106,9 +106,9 @@ CPUEntry::ComputeLoad()
 ThreadData*
 CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack)
 {
-	SpinLocker runQueueLocker(fCore->fQueueLock);
+	CoreRunQueueLocker _(fCore);
 
-	ThreadData* sharedThread = fCore->fRunQueue.PeekMaximum();
+	ThreadData* sharedThread = fCore->PeekThread();
 	ThreadData* pinnedThread = fRunQueue.PeekMaximum();
 
 	ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL);
@@ -132,16 +132,7 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack)
 	if (sharedPriority > pinnedPriority) {
 		sharedThread->fEnqueued = false;
 
-		fCore->fRunQueue.Remove(sharedThread);
+		fCore->Remove(sharedThread, sharedThread->fWentSleepCount == 0);
-		if (thread_is_idle_thread(sharedThread->GetThread())
-			|| fCore->fThreadList.Head() == sharedThread) {
-			atomic_add(&fCore->fStarvationCounter, 1);
-		}
-
-		if (sharedThread->fWentSleepCount == 0)
-			fCore->fThreadList.Remove(sharedThread);
-
-		atomic_add(&fCore->fThreadCount, -1);
 		return sharedThread;
 	}
 
@@ -259,15 +250,62 @@ CoreEntry::CoreEntry()
 
 
 void
-CoreEntry::UpdateLoad()
+CoreEntry::Init(int32 id, PackageEntry* package)
 {
-	ASSERT(!gSingleCore);
+	fCoreID = id;
+	fPackage = package;
+}
 
+
+void
+CoreEntry::PushFront(ThreadData* thread, int32 priority)
+{
+	fRunQueue.PushFront(thread, priority);
+	atomic_add(&fThreadCount, 1);
+}
+
+
+void
+CoreEntry::PushBack(ThreadData* thread, int32 priority)
+{
+	fRunQueue.PushBack(thread, priority);
+	fThreadList.Insert(thread);
+
+	atomic_add(&fThreadCount, 1);
+}
+
+
+void
+CoreEntry::Remove(ThreadData* thread, bool starving)
+{
+	if (thread_is_idle_thread(thread->GetThread())
+		|| fThreadList.Head() == thread) {
+		atomic_add(&fStarvationCounter, 1);
+	}
+	if (starving)
+		fThreadList.Remove(thread);
+	fRunQueue.Remove(thread);
+	atomic_add(&fThreadCount, -1);
+}
+
+
+inline ThreadData*
+CoreEntry::PeekThread() const
+{
+	return fRunQueue.PeekMaximum();
+}
+
+
+void
+CoreEntry::UpdateLoad(int32 delta)
+{
 	if (fCPUCount == 0) {
 		fLoad = 0;
 		return;
 	}
 
+	atomic_add(&fLoad, delta);
+
 	WriteSpinLocker coreLocker(gCoreHeapsLock);
 
 	int32 newKey = GetLoad();
@@ -310,6 +348,81 @@ CoreEntry::UpdateLoad()
 }
 
 
+void
+CoreEntry::AddCPU(CPUEntry* cpu)
+{
+	ASSERT(fCPUCount >= 0);
+	if (fCPUCount++ == 0) {
+		// core has been reenabled
+		fLoad = 0;
+		fHighLoad = false;
+		gCoreLoadHeap.Insert(this, 0);
+
+		fPackage->AddIdleCore(this);
+	}
+
+	fCPUHeap.Insert(cpu, B_IDLE_PRIORITY);
+}
+
+
+void
+CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
+{
+	ASSERT(fCPUCount > 0);
+	if (--fCPUCount == 0) {
+		// core has been disabled
+		if (fHighLoad) {
+			gCoreHighLoadHeap.ModifyKey(this, -1);
+			ASSERT(gCoreHighLoadHeap.PeekMinimum() == this);
+			gCoreHighLoadHeap.RemoveMinimum();
+		} else {
+			gCoreLoadHeap.ModifyKey(this, -1);
+			ASSERT(gCoreLoadHeap.PeekMinimum() == this);
+			gCoreLoadHeap.RemoveMinimum();
+		}
+
+		fPackage->RemoveIdleCore(this);
+
+		// get rid of threads
+		thread_map(CoreEntry::_UnassignThread, this);
+
+		fThreadCount = 0;
+		while (fRunQueue.PeekMaximum() != NULL) {
+			ThreadData* threadData = fRunQueue.PeekMaximum();
+
+			fRunQueue.Remove(threadData);
+			threadData->fEnqueued = false;
+
+			if (threadData->fWentSleepCount == 0)
+				fThreadList.Remove(threadData);
+			threadData->fWentSleepCount = -1;
+
+			ASSERT(threadData->Core() == NULL);
+			threadPostProcessing(threadData);
+		}
+	}
+
+	fCPUHeap.ModifyKey(cpu, THREAD_MAX_SET_PRIORITY + 1);
+	ASSERT(fCPUHeap.PeekMaximum() == cpu);
+	fCPUHeap.RemoveMaximum();
+
+	ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad);
+	fLoad -= cpu->fLoad;
+	ASSERT(fLoad >= 0);
+}
+
+
+/* static */ void
+CoreEntry::_UnassignThread(Thread* thread, void* data)
+{
+	CoreEntry* core = static_cast<CoreEntry*>(data);
+	ThreadData* threadData = thread->scheduler_data;
+
+	if (threadData->Core() == core && thread->pinned_to_cpu == 0)
+		threadData->UnassignCore();
+}
+
+
 CoreLoadHeap::CoreLoadHeap(int32 coreCount)
 	:
 	MinMaxHeap<CoreEntry, int32>(coreCount)
@@ -323,7 +436,7 @@ CoreLoadHeap::Dump()
 	CoreEntry* entry = PeekMinimum();
 	while (entry) {
 		int32 key = GetKey(entry);
-		kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->fCoreID,
+		kprintf("%4" B_PRId32 " %3" B_PRId32 "%%\n", entry->ID(),
 			entry->GetLoad() / 10);
 
 		RemoveMinimum();
@@ -420,6 +533,13 @@ PackageEntry::RemoveIdleCore(CoreEntry* core)
 }
 
 
+/* static */ void
+DebugDumper::DumpCoreRunQueue(CoreEntry* core)
+{
+	core->fRunQueue.Dump();
+}
+
+
 /* static */ void
 DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
 {
@@ -430,7 +550,7 @@ DebugDumper::DumpIdleCoresInPackage(PackageEntry* package)
 	if (iterator.HasNext()) {
 		while (iterator.HasNext()) {
 			CoreEntry* coreEntry = iterator.Next();
-			kprintf("%" B_PRId32 "%s", coreEntry->fCoreID,
+			kprintf("%" B_PRId32 "%s", coreEntry->ID(),
 				iterator.HasNext() ? ", " : "");
 		}
 	} else
@@ -445,10 +565,9 @@ dump_run_queue(int argc, char **argv)
 	int32 cpuCount = smp_get_num_cpus();
 	int32 coreCount = gCoreCount;
 
-	
 	for (int32 i = 0; i < coreCount; i++) {
 		kprintf("%sCore %" B_PRId32 " run queue:\n", i > 0 ? "\n" : "", i);
-		gCoreEntries[i].fRunQueue.Dump();
+		DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]);
 	}
 
 	for (int32 i = 0; i < cpuCount; i++) {
@@ -476,11 +595,11 @@ dump_cpu_heap(int argc, char** argv)
 	gCoreHighLoadHeap.Dump();
 
 	for (int32 i = 0; i < gCoreCount; i++) {
-		if (gCoreEntries[i].fCPUCount < 2)
+		if (gCoreEntries[i].CPUCount() < 2)
 			continue;
 
 		kprintf("\nCore %" B_PRId32 " heap:\n", i);
-		gCoreEntries[i].fCPUHeap.Dump();
+		gCoreEntries[i].CPUHeap()->Dump();
 	}
 
 	return 0;

src/system/kernel/scheduler/scheduler_cpu.h

@@ -9,6 +9,7 @@
 #include <OS.h>
 
 #include <thread.h>
+#include <util/AutoLock.h>
 #include <util/MinMaxHeap.h>
 
 #include <cpufreq.h>
@@ -24,6 +25,7 @@ namespace Scheduler {
 class DebugDumper;
 
 struct ThreadData;
+class ThreadProcessing;
 
 struct CPUEntry;
 struct CoreEntry;
@@ -77,15 +79,57 @@ public:
 						void			Dump();
 };
 
-struct CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
+class CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
-	DoublyLinkedListLinkImpl<CoreEntry> {
+	public DoublyLinkedListLinkImpl<CoreEntry> {
+public:
 										CoreEntry();
 
+						void			Init(int32 id, PackageEntry* package);
+
+	inline				int32			ID() const	{ return fCoreID; }
+	inline				PackageEntry*	Package() const	{ return fPackage; }
+	inline				int32			CPUCount() const
+											{ return fCPUCount; }
+
+	inline				void			LockCPUHeap();
+	inline				void			UnlockCPUHeap();
+
+	inline				CPUPriorityHeap*	CPUHeap();
+
+	inline				int32			ThreadCount() const
+											{ return fThreadCount; }
+
+	inline				void			LockRunQueue();
+	inline				void			UnlockRunQueue();
+
+						void			PushFront(ThreadData* thread,
+											int32 priority);
+						void			PushBack(ThreadData* thread,
+											int32 priority);
+						void			Remove(ThreadData* thread,
+											bool starving);
+	inline				ThreadData*		PeekThread() const;
+
+	inline				bigtime_t		GetActiveTime() const;
+	inline				void			IncreaseActiveTime(
+											bigtime_t activeTime);
+
 	inline				int32			GetLoad() const;
-						void			UpdateLoad();
+						void			UpdateLoad(int32 delta);
+
+	inline				int32			StarvationCounter() const;
+
+						void			AddCPU(CPUEntry* cpu);
+						void			RemoveCPU(CPUEntry* cpu,
+											ThreadProcessing&
+												threadPostProcessing);
 
 	static inline		CoreEntry*		GetCore(int32 cpu);
 
+private:
+	static				void			_UnassignThread(Thread* thread,
+											void* core);
+
 						int32			fCoreID;
 						PackageEntry*	fPackage;
 
@@ -101,12 +145,46 @@ struct CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
 						spinlock		fQueueLock;
 
 						bigtime_t		fActiveTime;
-						seqlock			fActiveTimeLock;
+	mutable				seqlock			fActiveTimeLock;
 
 						int32			fLoad;
 						bool			fHighLoad;
+
+						friend class DebugDumper;
 } CACHE_LINE_ALIGN;
 
+class CoreRunQueueLocking {
+public:
+	inline bool Lock(CoreEntry* core)
+	{
+		core->LockRunQueue();
+		return true;
+	}
+
+	inline void Unlock(CoreEntry* core)
+	{
+		core->UnlockRunQueue();
+	}
+};
+
+typedef AutoLocker<CoreEntry, CoreRunQueueLocking> CoreRunQueueLocker;
+
+class CoreCPUHeapLocking {
+public:
+	inline bool Lock(CoreEntry* core)
+	{
+		core->LockCPUHeap();
+		return true;
+	}
+
+	inline void Unlock(CoreEntry* core)
+	{
+		core->UnlockCPUHeap();
+	}
+};
+
+typedef AutoLocker<CoreEntry, CoreCPUHeapLocking> CoreCPUHeapLocker;
+
 class CoreLoadHeap : public MinMaxHeap<CoreEntry, int32> {
 public:
 										CoreLoadHeap() { }
@@ -167,6 +245,64 @@ extern rw_spinlock gIdlePackageLock;
 extern int32 gPackageCount;
 
 
+inline void
+CoreEntry::LockCPUHeap()
+{
+	acquire_spinlock(&fCPULock);
+}
+
+
+inline void
+CoreEntry::UnlockCPUHeap()
+{
+	release_spinlock(&fCPULock);
+}
+
+
+inline CPUPriorityHeap*
+CoreEntry::CPUHeap()
+{
+	return &fCPUHeap;
+}
+
+
+inline void
+CoreEntry::LockRunQueue()
+{
+	acquire_spinlock(&fQueueLock);
+}
+
+
+inline void
+CoreEntry::UnlockRunQueue()
+{
+	release_spinlock(&fQueueLock);
+}
+
+
+inline void
+CoreEntry::IncreaseActiveTime(bigtime_t activeTime)
+{
+	WriteSequentialLocker _(fActiveTimeLock);
+	fActiveTime += activeTime;
+}
+
+
+inline bigtime_t
+CoreEntry::GetActiveTime() const
+{
+	bigtime_t activeTime;
+
+	uint32 count;
+	do {
+		count = acquire_read_seqlock(&fActiveTimeLock);
+		activeTime = fActiveTime;
+	} while (!release_read_seqlock(&fActiveTimeLock, count));
+
+	return activeTime;
+}
+
+
 inline int32
 CoreEntry::GetLoad() const
 {
@@ -175,6 +311,13 @@ CoreEntry::GetLoad() const
 }
 
 
+inline int32
+CoreEntry::StarvationCounter() const
+{
+	return fStarvationCounter;
+}
+
+
 /* static */ inline CoreEntry*
 CoreEntry::GetCore(int32 cpu)
 {

src/system/kernel/scheduler/scheduler_thread.cpp

@@ -65,7 +65,7 @@ ThreadData::Dump() const
 	kprintf("\twent_sleep_active:\t%" B_PRId64 "\n", fWentSleepActive);
 	kprintf("\twent_sleep_count:\t%" B_PRId32 "\n", fWentSleepCount);
 	kprintf("\tcore:\t\t\t%" B_PRId32 "\n",
-		fCore != NULL ? fCore->fCoreID : -1);
+		fCore != NULL ? fCore->ID() : -1);
 	if (fCore != NULL && HasCacheExpired())
 		kprintf("\tcache affinity has expired\n");
 }
@@ -108,7 +108,7 @@ ThreadData::ComputeQuantum()
 	quantum += fStolenTime;
 	fStolenTime = 0;
 
-	int32 threadCount = (fCore->fThreadCount + 1) / fCore->fCPUCount;
+	int32 threadCount = (fCore->ThreadCount() + 1) / fCore->CPUCount();
 	threadCount = max_c(threadCount, 1);
 
 	quantum = std::min(gCurrentMode->maximum_latency / threadCount, quantum);
@@ -137,7 +137,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const
 	if (fThread->previous_cpu != NULL) {
 		CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num];
 		if (previousCPU->fCore == core) {
-			SpinLocker cpuLocker(core->fCPULock);
+			CoreCPUHeapLocker _(core);
 			if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) {
 				previousCPU->UpdatePriority(threadPriority);
 				rescheduleNeeded = true;
@@ -146,8 +146,8 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const
 		}
 	}
 
-	SpinLocker cpuLocker(core->fCPULock);
+	CoreCPUHeapLocker _(core);
-	CPUEntry* cpu = core->fCPUHeap.PeekMinimum();
+	CPUEntry* cpu = core->CPUHeap()->PeekMinimum();
 	ASSERT(cpu != NULL);
 
 	if (CPUPriorityHeap::GetKey(cpu) < threadPriority) {
@@ -195,3 +195,8 @@ ThreadData::_ScaleQuantum(bigtime_t maxQuantum, bigtime_t minQuantum,
 	return maxQuantum - result;
 }
 
+
+ThreadProcessing::~ThreadProcessing()
+{
+}
+

src/system/kernel/scheduler/scheduler_thread.h

@@ -93,6 +93,13 @@ private:
 			CoreEntry*	fCore;
 };
 
+class ThreadProcessing {
+public:
+	virtual				~ThreadProcessing();
+
+	virtual	void		operator()(ThreadData* thread) = 0;
+};
+
 
 inline bool
 ThreadData::HasCacheExpired() const
@@ -166,7 +173,7 @@ ThreadData::ShouldCancelPenalty() const
 	if (fCore == NULL)
 		return false;
 
-	return atomic_get(&fCore->fStarvationCounter) != fWentSleepCount
+	return fCore->StarvationCounter() != fWentSleepCount
 		&& system_time() - fWentSleep > gCurrentMode->base_quantum;
 }
 
@@ -177,13 +184,8 @@ ThreadData::GoesAway()
 	fLastInterruptTime = 0;
 
 	fWentSleep = system_time();
-	fWentSleepCount = atomic_get(&fCore->fStarvationCounter);
+	fWentSleepCount = fCore->StarvationCounter();
-
+	fWentSleepActive = fCore->GetActiveTime();
-	uint32 count;
-	do {
-		count = acquire_read_seqlock(&fCore->fActiveTimeLock);
-		fWentSleepActive = fCore->fActiveTime;
-	} while (!release_read_seqlock(&fCore->fActiveTimeLock, count));
 }
 
 
@@ -195,7 +197,7 @@ ThreadData::PutBack()
 
 	int32 priority = GetEffectivePriority();
 
-	SpinLocker runQueueLocker(fCore->fQueueLock);
+	CoreRunQueueLocker _(fCore);
 	ASSERT(!fEnqueued);
 	fEnqueued = true;
 	if (fThread->pinned_to_cpu > 0) {
@@ -203,10 +205,9 @@ ThreadData::PutBack()
 
 		CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num];
 		cpu->fRunQueue.PushFront(this, priority);
-	} else {
+	} else
-		fCore->fRunQueue.PushFront(this, priority);
+		fCore->PushFront(this, priority);
-		atomic_add(&fCore->fThreadCount, 1);
+	fCore->UnlockRunQueue();
-	}
 }
 
 
@@ -219,7 +220,7 @@ ThreadData::Enqueue()
 
 	int32 priority = GetEffectivePriority();
 
-	SpinLocker runQueueLocker(fCore->fQueueLock);
+	CoreRunQueueLocker _(fCore);
 	ASSERT(!fEnqueued);
 	fEnqueued = true;
 	if (fThread->pinned_to_cpu > 0) {
@@ -227,19 +228,15 @@ ThreadData::Enqueue()
 
 		CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
 		cpu->fRunQueue.PushBack(this, priority);
-	} else {
+	} else
-		fCore->fRunQueue.PushBack(this, priority);
+		fCore->PushBack(this, priority);
-		fCore->fThreadList.Insert(this);
-
-		atomic_add(&fCore->fThreadCount, 1);
-	}
 }
 
 
 inline bool
 ThreadData::Dequeue()
 {
-	SpinLocker runQueueLocker(fCore->fQueueLock);
+	CoreRunQueueLocker _(fCore);
 	if (!fEnqueued)
 		return false;
 
@@ -250,12 +247,8 @@ ThreadData::Dequeue()
 		CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
 		cpu->fRunQueue.Remove(this);
 	} else {
-		fCore->fRunQueue.Remove(this);
-
 		ASSERT(fWentSleepCount < 1);
-		if (fWentSleepCount == 0)
+		fCore->Remove(this, fWentSleepCount == 0);
-			fCore->fThreadList.Remove(this);
-		atomic_add(&fCore->fThreadCount, -1);
 	}
 
 	return true;
@@ -268,9 +261,8 @@ ThreadData::UpdateActivity(bigtime_t active)
 	fMeasureActiveTime += active;
 	gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active;
 
-	WriteSequentialLocker locker(fCore->fActiveTimeLock);
+	fCore->IncreaseActiveTime(active);
-	fCore->fActiveTime += active;
+
-	locker.Unlock();
 }