scheduler: Protect per CPU run queue with its own lock

src/system/kernel/scheduler/scheduler.cpp

@@ -517,7 +517,7 @@ reschedule(int32 nextState)
 	ThreadData* nextThreadData;
 	if (gCPU[thisCPU].disabled) {
 		if (!thread_is_idle_thread(oldThread)) {
-			CoreRunQueueLocker _(core);
+			CPURunQueueLocker _(cpu);
 
 			nextThreadData = cpu->PeekIdleThread();
 			cpu->Remove(nextThreadData);

src/system/kernel/scheduler/scheduler_cpu.cpp

@@ -56,6 +56,7 @@ CPUEntry::CPUEntry()
 	fMeasureTime(0)
 {
 	B_INITIALIZE_RW_SPINLOCK(&fSchedulerModeLock);
+	B_INITIALIZE_SPINLOCK(&fQueueLock);
 }
 
 
@@ -185,25 +186,26 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack)
 {
 	SCHEDULER_ENTER_FUNCTION();
 
-	CoreRunQueueLocker _(fCore);
+	int32 oldPriority = -1;
+	if (oldThread != NULL)
+		oldPriority = oldThread->GetEffectivePriority();
+
+	CPURunQueueLocker cpuLocker(this);
 
-	ThreadData* sharedThread = fCore->PeekThread();
 	ThreadData* pinnedThread = fRunQueue.PeekMaximum();
-
-	ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL);
-
 	int32 pinnedPriority = -1;
 	if (pinnedThread != NULL)
 		pinnedPriority = pinnedThread->GetEffectivePriority();
 
+	CoreRunQueueLocker coreLocker(fCore);
+
+	ThreadData* sharedThread = fCore->PeekThread();
+	ASSERT(sharedThread != NULL || pinnedThread != NULL || oldThread != NULL);
+
 	int32 sharedPriority = -1;
 	if (sharedThread != NULL)
 		sharedPriority = sharedThread->GetEffectivePriority();
 
-	int32 oldPriority = -1;
-	if (oldThread != NULL)
-		oldPriority = oldThread->GetEffectivePriority();
-
 	int32 rest = std::max(pinnedPriority, sharedPriority);
 	if (oldPriority > rest || (!putAtBack && oldPriority == rest))
 		return oldThread;
@@ -213,6 +215,8 @@ CPUEntry::ChooseNextThread(ThreadData* oldThread, bool putAtBack)
 		return sharedThread;
 	}
 
+	coreLocker.Unlock();
+
 	Remove(pinnedThread);
 	return pinnedThread;
 }

src/system/kernel/scheduler/scheduler_cpu.h

@@ -60,6 +60,9 @@ public:
 	inline				void			LockScheduler();
 	inline				void			UnlockScheduler();
 
+	inline				void			LockRunQueue();
+	inline				void			UnlockRunQueue();
+
 						void			PushFront(ThreadData* thread,
 											int32 priority);
 						void			PushBack(ThreadData* thread,
@@ -91,6 +94,7 @@ private:
 						rw_spinlock 	fSchedulerModeLock;
 
 						ThreadRunQueue	fRunQueue;
+						spinlock		fQueueLock;
 
 						int32			fLoad;
 
@@ -108,6 +112,22 @@ public:
 						void			Dump();
 };
 
+class CPURunQueueLocking {
+public:
+	inline bool Lock(CPUEntry* cpu)
+	{
+		cpu->LockRunQueue();
+		return true;
+	}
+
+	inline void Unlock(CPUEntry* cpu)
+	{
+		cpu->UnlockRunQueue();
+	}
+};
+
+typedef AutoLocker<CPUEntry, CPURunQueueLocking> CPURunQueueLocker;
+
 class CoreEntry : public MinMaxHeapLinkImpl<CoreEntry, int32>,
 	public DoublyLinkedListLinkImpl<CoreEntry> {
 public:
@@ -309,6 +329,22 @@ CPUEntry::UnlockScheduler()
 }
 
 
+inline void
+CPUEntry::LockRunQueue()
+{
+	SCHEDULER_ENTER_FUNCTION();
+	acquire_spinlock(&fQueueLock);
+}
+
+
+inline void
+CPUEntry::UnlockRunQueue()
+{
+	SCHEDULER_ENTER_FUNCTION();
+	release_spinlock(&fQueueLock);
+}
+
+
 /* static */ inline CPUEntry*
 CPUEntry::GetCPU(int32 cpu)
 {

src/system/kernel/scheduler/scheduler_thread.h

@@ -234,16 +234,22 @@ ThreadData::PutBack()
 
 	int32 priority = GetEffectivePriority();
 
-	CoreRunQueueLocker _(fCore);
-	ASSERT(!fEnqueued);
-	fEnqueued = true;
 	if (fThread->pinned_to_cpu > 0) {
 		ASSERT(fThread->cpu != NULL);
-
 		CPUEntry* cpu = CPUEntry::GetCPU(fThread->cpu->cpu_num);
+
+		CPURunQueueLocker _(cpu);
+		ASSERT(!fEnqueued);
+		fEnqueued = true;
+
 		cpu->PushFront(this, priority);
-	} else
+	} else {
+		CoreRunQueueLocker _(fCore);
+		ASSERT(!fEnqueued);
+		fEnqueued = true;
+
 		fCore->PushFront(this, priority);
+	}
 }
 
 
@@ -258,16 +264,22 @@ ThreadData::Enqueue()
 
 	int32 priority = GetEffectivePriority();
 
-	CoreRunQueueLocker _(fCore);
-	ASSERT(!fEnqueued);
-	fEnqueued = true;
 	if (fThread->pinned_to_cpu > 0) {
 		ASSERT(fThread->previous_cpu != NULL);
-
 		CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
+
+		CPURunQueueLocker _(cpu);
+		ASSERT(!fEnqueued);
+		fEnqueued = true;
+
 		cpu->PushBack(this, priority);
-	} else
+	} else {
+		CoreRunQueueLocker _(fCore);
+		ASSERT(!fEnqueued);
+		fEnqueued = true;
+
 		fCore->PushBack(this, priority);
+	}
 }
 
 
@@ -276,20 +288,23 @@ ThreadData::Dequeue()
 {
 	SCHEDULER_ENTER_FUNCTION();
 
+	if (fThread->pinned_to_cpu > 0) {
+		ASSERT(fThread->previous_cpu != NULL);
+		CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
+
+		CPURunQueueLocker _(cpu);
+		if (!fEnqueued)
+			return false;
+		cpu->Remove(this);
+		ASSERT(!fEnqueued);
+		return true;
+	}
+
 	CoreRunQueueLocker _(fCore);
 	if (!fEnqueued)
 		return false;
-
+	ASSERT(fWentSleepCount < 1);
-	if (fThread->pinned_to_cpu > 0) {
+	fCore->Remove(this);
-		ASSERT(fThread->previous_cpu != NULL);
-
-		CPUEntry* cpu = CPUEntry::GetCPU(fThread->previous_cpu->cpu_num);
-		cpu->Remove(this);
-	} else {
-		ASSERT(fWentSleepCount < 1);
-		fCore->Remove(this);
-	}
-
 	ASSERT(!fEnqueued);
 	return true;
 }