scheduler: Encapsulate CPUEntry fields

src/system/kernel/scheduler/low_latency.cpp

@@ -132,7 +132,7 @@ rebalance_irqs(bool idle)
 		other = gCoreHighLoadHeap.PeekMinimum();
 	coreLocker.Unlock();
 
-	int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
+	int32 newCPU = other->CPUHeap()->PeekMinimum()->ID();
 
 	ASSERT(other != NULL);
 

src/system/kernel/scheduler/power_saving.cpp

@@ -164,7 +164,7 @@ pack_irqs()
 		irq_assignment* irq = (irq_assignment*)list_get_first_item(&cpu->irqs);
 		locker.Unlock();
 
-		int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->fCPUNumber;
+		int32 newCPU = smallTaskCore->CPUHeap()->PeekMinimum()->ID();
 
 		if (newCPU != cpu->cpu_num)
 			assign_io_interrupt_to_cpu(irq->irq, newCPU);
@@ -207,7 +207,7 @@ rebalance_irqs(bool idle)
 	coreLocker.Unlock();
 	if (other == NULL)
 		return;
-	int32 newCPU = other->CPUHeap()->PeekMinimum()->fCPUNumber;
+	int32 newCPU = other->CPUHeap()->PeekMinimum()->ID();
 
 	CoreEntry* core = CoreEntry::GetCore(smp_get_current_cpu());
 	if (other == core)

src/system/kernel/scheduler/scheduler.cpp

@@ -39,26 +39,60 @@
 namespace Scheduler {
 
 
-class SchedulerModeLocker : public ReadSpinLocker {
+class SchedulerModeLocking {
+public:
+	bool Lock(int* /* lockable */)
+	{
+		CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler();
+		return true;
+	}
+
+	void Unlock(int* /* lockable */)
+	{
+		CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler();
+	}
+};
+
+class SchedulerModeLocker :
+	public AutoLocker<int, SchedulerModeLocking> {
 public:
 	SchedulerModeLocker(bool alreadyLocked = false, bool lockIfNotLocked = true)
 		:
-		ReadSpinLocker(gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock,
+		AutoLocker<int, SchedulerModeLocking>(NULL, alreadyLocked,
-			alreadyLocked, lockIfNotLocked)
+			lockIfNotLocked)
 	{
 	}
 };
 
-class InterruptsSchedulerModeLocker : public InterruptsReadSpinLocker {
+class InterruptsSchedulerModeLocking {
+public:
+	bool Lock(int* lockable)
+	{
+		*lockable = disable_interrupts();
+		CPUEntry::GetCPU(smp_get_current_cpu())->EnterScheduler();
+		return true;
+	}
+
+	void Unlock(int* lockable)
+	{
+		CPUEntry::GetCPU(smp_get_current_cpu())->ExitScheduler();
+		restore_interrupts(*lockable);
+	}
+};
+
+class InterruptsSchedulerModeLocker :
+	public AutoLocker<int, InterruptsSchedulerModeLocking> {
 public:
 	InterruptsSchedulerModeLocker(bool alreadyLocked = false,
 		bool lockIfNotLocked = true)
 		:
-		InterruptsReadSpinLocker(
+		AutoLocker<int, InterruptsSchedulerModeLocking>(&fState, alreadyLocked,
-			gCPUEntries[smp_get_current_cpu()].fSchedulerModeLock,
+			lockIfNotLocked)
-			alreadyLocked, lockIfNotLocked)
 	{
 	}
+
+private:
+	int		fState;
 };
 
 class InterruptsBigSchedulerLocking {
@@ -67,14 +101,14 @@ public:
 	{
 		*lockable = disable_interrupts();
 		for (int32 i = 0; i < smp_get_num_cpus(); i++)
-			acquire_write_spinlock(&gCPUEntries[i].fSchedulerModeLock);
+			CPUEntry::GetCPU(i)->LockScheduler();
 		return true;
 	}
 
 	void Unlock(int* lockable)
 	{
 		for (int32 i = 0; i < smp_get_num_cpus(); i++)
-			release_write_spinlock(&gCPUEntries[i].fSchedulerModeLock);
+			CPUEntry::GetCPU(i)->UnlockScheduler();
 		restore_interrupts(*lockable);
 	}
 };
@@ -193,10 +227,10 @@ enqueue(Thread* thread, bool newOne)
 	if (threadPriority > heapPriority
 		|| (threadPriority == heapPriority && rescheduleNeeded)) {
 
-		if (targetCPU->fCPUNumber == smp_get_current_cpu())
+		if (targetCPU->ID() == smp_get_current_cpu())
-			gCPU[targetCPU->fCPUNumber].invoke_scheduler = true;
+			gCPU[targetCPU->ID()].invoke_scheduler = true;
 		else {
-			smp_send_ici(targetCPU->fCPUNumber, SMP_MSG_RESCHEDULE, 0, 0, 0,
+			smp_send_ici(targetCPU->ID(), SMP_MSG_RESCHEDULE, 0, 0, 0,
 				NULL, SMP_MSG_FLAG_ASYNC);
 		}
 	}
@@ -484,8 +518,8 @@ reschedule(int32 nextState)
 		if (!thread_is_idle_thread(oldThread)) {
 			CoreRunQueueLocker _(core);
 
-			nextThreadData = cpu->fRunQueue.GetHead(B_IDLE_PRIORITY);
+			nextThreadData = cpu->PeekIdleThread();
-		 	cpu->fRunQueue.Remove(nextThreadData);
+			cpu->Remove(nextThreadData);
 			nextThreadData->fEnqueued = false;
 
 			putOldThreadAtBack = oldThread->pinned_to_cpu == 0;
@@ -662,14 +696,13 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
 	gCurrentMode->set_cpu_enabled(cpuID, enabled);
 
 	CPUEntry* cpu = &gCPUEntries[cpuID];
-	CoreEntry* core = cpu->fCore;
+	CoreEntry* core = cpu->Core();
 
 	int32 oldCPUCount = core->CPUCount();
 	ASSERT(oldCPUCount >= 0);
-	if (enabled) {
+	if (enabled)
-		cpu->fLoad = 0;
+		cpu->Start();
-		core->AddCPU(cpu);
+	else {
-	} else {
 		cpu->UpdatePriority(B_IDLE_PRIORITY);
 
 		ThreadEnqueuer enqueuer;
@@ -679,21 +712,7 @@ scheduler_set_cpu_enabled(int32 cpuID, bool enabled)
 	gCPU[cpuID].disabled = !enabled;
 
 	if (!enabled) {
-		cpu_ent* entry = &gCPU[cpuID];
+		cpu->Stop();
-
-		// get rid of irqs
-		SpinLocker locker(entry->irqs_lock);
-		irq_assignment* irq
-			= (irq_assignment*)list_get_first_item(&entry->irqs);
-		while (irq != NULL) {
-			locker.Unlock();
-
-			assign_io_interrupt_to_cpu(irq->irq, -1);
-
-			locker.Lock();
-			irq = (irq_assignment*)list_get_first_item(&entry->irqs);
-		}
-		locker.Unlock();
 
 		// don't wait until the thread quantum ends
 		if (smp_get_current_cpu() != cpuID) {
@@ -808,9 +827,7 @@ init()
 
 		package->Init(sCPUToPackage[i]);
 		core->Init(sCPUToCore[i], package);
-
+		gCPUEntries[i].Init(i, core);
-		gCPUEntries[i].fCPUNumber = i;
-		gCPUEntries[i].fCore = core;
 
 		core->AddCPU(&gCPUEntries[i]);
 	}

src/system/kernel/scheduler/scheduler_cpu.cpp

@@ -18,6 +18,7 @@ using namespace Scheduler;
 
 class Scheduler::DebugDumper {
 public:
+	static	void		DumpCPURunQueue(CPUEntry* cpu);
 	static	void		DumpCoreRunQueue(CoreEntry* core);
 	static	void		DumpIdleCoresInPackage(PackageEntry* package);
 
@@ -58,6 +59,78 @@ CPUEntry::CPUEntry()
 }
 
 
+void
+CPUEntry::Init(int32 id, CoreEntry* core)
+{
+	fCPUNumber = id;
+	fCore = core;
+}
+
+
+void
+CPUEntry::Start()
+{
+	fLoad = 0;
+	fCore->AddCPU(this);
+}
+
+
+void
+CPUEntry::Stop()
+{
+	cpu_ent* entry = &gCPU[fCPUNumber];
+
+	// get rid of irqs
+	SpinLocker locker(entry->irqs_lock);
+	irq_assignment* irq
+		= (irq_assignment*)list_get_first_item(&entry->irqs);
+	while (irq != NULL) {
+		locker.Unlock();
+
+		assign_io_interrupt_to_cpu(irq->irq, -1);
+
+		locker.Lock();
+		irq = (irq_assignment*)list_get_first_item(&entry->irqs);
+	}
+	locker.Unlock();
+}
+
+
+void
+CPUEntry::PushFront(ThreadData* thread, int32 priority)
+{
+	fRunQueue.PushFront(thread, priority);
+}
+
+
+void
+CPUEntry::PushBack(ThreadData* thread, int32 priority)
+{
+	fRunQueue.PushBack(thread, priority);
+}
+
+
+void
+CPUEntry::Remove(ThreadData* thread)
+{
+	fRunQueue.Remove(thread);
+}
+
+
+inline ThreadData*
+CPUEntry::PeekThread() const
+{
+	return fRunQueue.PeekMaximum();
+}
+
+
+ThreadData*
+CPUEntry::PeekIdleThread() const
+{
+	return fRunQueue.GetHead(B_IDLE_PRIORITY);
+}
+
+
 void
 CPUEntry::UpdatePriority(int32 priority)
 {
@@ -213,10 +286,10 @@ CPUPriorityHeap::Dump()
 	kprintf("cpu priority load\n");
 	CPUEntry* entry = PeekMinimum();
 	while (entry) {
-		int32 cpu = entry->fCPUNumber;
+		int32 cpu = entry->ID();
 		int32 key = GetKey(entry);
 		kprintf("%3" B_PRId32 " %8" B_PRId32 " %3" B_PRId32 "%%\n", cpu, key,
-			entry->fLoad / 10);
+			entry->GetLoad() / 10);
 
 		RemoveMinimum();
 		sDebugCPUHeap.Insert(entry, key);
@@ -406,8 +479,8 @@ CoreEntry::RemoveCPU(CPUEntry* cpu, ThreadProcessing& threadPostProcessing)
 	ASSERT(fCPUHeap.PeekMaximum() == cpu);
 	fCPUHeap.RemoveMaximum();
 
-	ASSERT(cpu->fLoad >= 0 && cpu->fLoad <= kMaxLoad);
+	ASSERT(cpu->GetLoad() >= 0 && cpu->GetLoad() <= kMaxLoad);
-	fLoad -= cpu->fLoad;
+	fLoad -= cpu->GetLoad();
 	ASSERT(fLoad >= 0);
 }
 
@@ -533,6 +606,19 @@ PackageEntry::RemoveIdleCore(CoreEntry* core)
 }
 
 
+/* static */ void
+DebugDumper::DumpCPURunQueue(CPUEntry* cpu)
+{
+	ThreadRunQueue::ConstIterator iterator = cpu->fRunQueue.GetConstIterator();
+
+	if (iterator.HasNext()
+		&& !thread_is_idle_thread(iterator.Next()->GetThread())) {
+		kprintf("\nCPU %" B_PRId32 " run queue:\n", cpu->ID());
+		cpu->fRunQueue.Dump();
+	}
+}
+
+
 /* static */ void
 DebugDumper::DumpCoreRunQueue(CoreEntry* core)
 {
@@ -570,17 +656,8 @@ dump_run_queue(int argc, char **argv)
 		DebugDumper::DumpCoreRunQueue(&gCoreEntries[i]);
 	}
 
-	for (int32 i = 0; i < cpuCount; i++) {
+	for (int32 i = 0; i < cpuCount; i++)
-		CPUEntry* cpu = &gCPUEntries[i];
+		DebugDumper::DumpCPURunQueue(&gCPUEntries[i]);
-		ThreadRunQueue::ConstIterator iterator
-			= cpu->fRunQueue.GetConstIterator();
-
-		if (iterator.HasNext()
-			&& !thread_is_idle_thread(iterator.Next()->GetThread())) {
-			kprintf("\nCPU %" B_PRId32 " run queue:\n", i);
-			cpu->fRunQueue.Dump();
-		}
-	}
 
 	return 0;
 }

src/system/kernel/scheduler/scheduler_cpu.h

@@ -40,11 +40,38 @@ public:
 						void			Dump() const;
 };
 
-struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
+class CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
+public:
 										CPUEntry();
 
+						void			Init(int32 id, CoreEntry* core);
+
+	inline				int32			ID() const	{ return fCPUNumber; }
+	inline				CoreEntry*		Core() const	{ return fCore; }
+
+						void			Start();
+						void			Stop();
+
+	inline				void			EnterScheduler();
+	inline				void			ExitScheduler();
+
+	inline				void			LockScheduler();
+	inline				void			UnlockScheduler();
+
+						void			PushFront(ThreadData* thread,
+											int32 priority);
+						void			PushBack(ThreadData* thread,
+											int32 priority);
+						void			Remove(ThreadData* thread);
+	inline				ThreadData*		PeekThread() const;
+						ThreadData*		PeekIdleThread() const;
+
 						void			UpdatePriority(int32 priority);
 
+	inline				void			IncreaseActiveTime(
+											bigtime_t activeTime);
+
+	inline				int32			GetLoad() const	{ return fLoad; }
 						void			ComputeLoad();
 
 						ThreadData*		ChooseNextThread(ThreadData* oldThread,
@@ -53,6 +80,12 @@ struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
 						void			TrackActivity(ThreadData* oldThreadData,
 											ThreadData* nextThreadData);
 
+	static inline		CPUEntry*		GetCPU(int32 cpu);
+
+private:
+	inline				void			_RequestPerformanceLevel(
+											ThreadData* threadData);
+
 						int32			fCPUNumber;
 						CoreEntry*		fCore;
 
@@ -65,10 +98,7 @@ struct CPUEntry : public MinMaxHeapLinkImpl<CPUEntry, int32> {
 						bigtime_t		fMeasureActiveTime;
 						bigtime_t		fMeasureTime;
 
-private:
+						friend class DebugDumper;
-	inline				void			_RequestPerformanceLevel(
-											ThreadData* threadData);
-
 } CACHE_LINE_ALIGN;
 
 class CPUPriorityHeap : public MinMaxHeap<CPUEntry, int32> {
@@ -245,6 +275,48 @@ extern rw_spinlock gIdlePackageLock;
 extern int32 gPackageCount;
 
 
+inline void
+CPUEntry::EnterScheduler()
+{
+	acquire_read_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::ExitScheduler()
+{
+	release_read_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::LockScheduler()
+{
+	acquire_write_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::UnlockScheduler()
+{
+	release_write_spinlock(&fSchedulerModeLock);
+}
+
+
+inline void
+CPUEntry::IncreaseActiveTime(bigtime_t activeTime)
+{
+	fMeasureActiveTime += activeTime;
+}
+
+
+/* static */ inline CPUEntry*
+CPUEntry::GetCPU(int32 cpu)
+{
+	return &gCPUEntries[cpu];
+}
+
+
 inline void
 CoreEntry::LockCPUHeap()
 {
@@ -321,7 +393,7 @@ CoreEntry::StarvationCounter() const
 /* static */ inline CoreEntry*
 CoreEntry::GetCore(int32 cpu)
 {
-	return gCPUEntries[cpu].fCore;
+	return gCPUEntries[cpu].Core();
 }
 
 

src/system/kernel/scheduler/scheduler_thread.cpp

@@ -77,7 +77,7 @@ ThreadData::ChooseCoreAndCPU(CoreEntry*& targetCore, CPUEntry*& targetCPU)
 	bool rescheduleNeeded = false;
 
 	if (targetCore == NULL && targetCPU != NULL)
-		targetCore = targetCPU->fCore;
+		targetCore = targetCPU->Core();
 	else if (targetCore != NULL && targetCPU == NULL)
 		targetCPU = _ChooseCPU(targetCore, rescheduleNeeded);
 	else if (targetCore == NULL && targetCPU == NULL) {
@@ -136,7 +136,7 @@ ThreadData::_ChooseCPU(CoreEntry* core, bool& rescheduleNeeded) const
 
 	if (fThread->previous_cpu != NULL) {
 		CPUEntry* previousCPU = &gCPUEntries[fThread->previous_cpu->cpu_num];
-		if (previousCPU->fCore == core) {
+		if (previousCPU->Core() == core) {
 			CoreCPUHeapLocker _(core);
 			if (CPUPriorityHeap::GetKey(previousCPU) < threadPriority) {
 				previousCPU->UpdatePriority(threadPriority);

src/system/kernel/scheduler/scheduler_thread.h

@@ -203,8 +203,8 @@ ThreadData::PutBack()
 	if (fThread->pinned_to_cpu > 0) {
 		ASSERT(fThread->cpu != NULL);
 
-		CPUEntry* cpu = &gCPUEntries[fThread->cpu->cpu_num];
+		CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num);
-		cpu->fRunQueue.PushFront(this, priority);
+		cpu->PushFront(this, priority);
 	} else
 		fCore->PushFront(this, priority);
 	fCore->UnlockRunQueue();
@@ -226,8 +226,8 @@ ThreadData::Enqueue()
 	if (fThread->pinned_to_cpu > 0) {
 		ASSERT(fThread->previous_cpu != NULL);
 
-		CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
+		CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
-		cpu->fRunQueue.PushBack(this, priority);
+		cpu->PushBack(this, priority);
 	} else
 		fCore->PushBack(this, priority);
 }
@@ -244,8 +244,8 @@ ThreadData::Dequeue()
 	if (fThread->pinned_to_cpu > 0) {
 		ASSERT(fThread->previous_cpu != NULL);
 
-		CPUEntry* cpu = &gCPUEntries[fThread->previous_cpu->cpu_num];
+		CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
-		cpu->fRunQueue.Remove(this);
+		cpu->Remove(this);
 	} else {
 		ASSERT(fWentSleepCount < 1);
 		fCore->Remove(this, fWentSleepCount == 0);
@@ -259,10 +259,8 @@ inline void
 ThreadData::UpdateActivity(bigtime_t active)
 {
 	fMeasureActiveTime += active;
-	gCPUEntries[smp_get_current_cpu()].fMeasureActiveTime += active;
+	CPUEntry::GetCPU(smp_get_current_cpu())->IncreaseActiveTime(active);
-
 	fCore->IncreaseActiveTime(active);
-
 }