From 03451e4cc166588ed1b399bb97acaf3a043283ff Mon Sep 17 00:00:00 2001
From: Pawel Dziepak <pdziepak@quarnos.org>
Date: Thu, 23 Jan 2014 23:53:35 +0100
Subject: [PATCH] kernel: Fix deadlock with thread sending signal to itself

UserEvent can be fired from scheduler_reschedule() i.e. while holding current
thread scheduler_lock. If the current thread goes sleep and during reschedule
one of its timers sends a signel to it, then scheduler_enqueue_in_run_queue()
attempts to acquire again its scheduler_lock resulting in a deadlock.

There was also a minor issue with both scheduler_reschedule() and
scheduler_enqueue_in_run_queue() acquiring current CPU scheduler mode lock.
---
 headers/private/kernel/UserEvent.h | 11 +++++---
 src/system/kernel/UserEvent.cpp    | 41 ++++++++++++++++++++----------
 2 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/headers/private/kernel/UserEvent.h b/headers/private/kernel/UserEvent.h
index 34086bc57e..8e0a931afe 100644
--- a/headers/private/kernel/UserEvent.h
+++ b/headers/private/kernel/UserEvent.h
@@ -28,11 +28,13 @@ struct UserEvent {
 };
 
 
-struct SignalEvent : UserEvent {
+struct SignalEvent : UserEvent, private DPCCallback {
 	virtual						~SignalEvent();
 
 			void				SetUserValue(union sigval userValue);
 
+	virtual	status_t			Fire();
+
 protected:
 			struct EventSignal;
 
@@ -41,6 +43,7 @@ protected:
 
 protected:
 			EventSignal*		fSignal;
+			int32				fPendingDPC;
 };
 
 
@@ -48,7 +51,8 @@ struct TeamSignalEvent : SignalEvent {
 	static	TeamSignalEvent*	Create(Team* team, uint32 signalNumber,
 									int32 signalCode, int32 errorCode);
 
-	virtual	status_t			Fire();
+protected:
+	virtual	void				DoDPC(DPCQueue* queue);
 
 private:
 								TeamSignalEvent(Team* team,
@@ -64,7 +68,8 @@ struct ThreadSignalEvent : SignalEvent {
 									int32 signalCode, int32 errorCode,
 									pid_t sendingTeam);
 
-	virtual	status_t			Fire();
+protected:
+	virtual	void				DoDPC(DPCQueue* queue);
 
 private:
 								ThreadSignalEvent(Thread* thread,
diff --git a/src/system/kernel/UserEvent.cpp b/src/system/kernel/UserEvent.cpp
index f0c49b6f58..46acc63344 100644
--- a/src/system/kernel/UserEvent.cpp
+++ b/src/system/kernel/UserEvent.cpp
@@ -56,7 +56,8 @@ private:
 
 SignalEvent::SignalEvent(EventSignal* signal)
 	:
-	fSignal(signal)
+	fSignal(signal),
+	fPendingDPC(0)
 {
 }
 
@@ -74,6 +75,24 @@ SignalEvent::SetUserValue(union sigval userValue)
 }
 
 
+status_t
+SignalEvent::Fire()
+{
+	bool wasPending = atomic_get_and_set(&fPendingDPC, 1) != 0;
+	if (wasPending)
+		return B_BUSY;
+
+	if (fSignal->MarkUsed()) {
+		atomic_set(&fPendingDPC, 0);
+		return B_BUSY;
+	}
+
+	DPCQueue::DefaultQueue(B_NORMAL_PRIORITY)->Add(this);
+
+	return B_OK;
+}
+
+
 // #pragma mark - TeamSignalEvent
 
 
@@ -106,12 +125,9 @@ TeamSignalEvent::Create(Team* team, uint32 signalNumber, int32 signalCode,
 }
 
 
-status_t
-TeamSignalEvent::Fire()
+void
+TeamSignalEvent::DoDPC(DPCQueue* queue)
 {
-	if (fSignal->MarkUsed())
-		return B_BUSY;
-
 	fSignal->AcquireReference();
 		// one reference is transferred to send_signal_to_team_locked
 
@@ -125,7 +141,8 @@ TeamSignalEvent::Fire()
 	if (error != B_OK || !fSignal->IsPending())
 		fSignal->SetUnused();
 
-	return error;
+	// We're no longer queued in the DPC queue, so we can be reused.
+	atomic_set(&fPendingDPC, 0);
 }
 
 
@@ -161,12 +178,9 @@ ThreadSignalEvent::Create(Thread* thread, uint32 signalNumber, int32 signalCode,
 }
 
 
-status_t
-ThreadSignalEvent::Fire()
+void
+ThreadSignalEvent::DoDPC(DPCQueue* queue)
 {
-	if (fSignal->MarkUsed())
-		return B_BUSY;
-
 	fSignal->AcquireReference();
 		// one reference is transferred to send_signal_to_team_locked
 	InterruptsReadSpinLocker teamLocker(fThread->team_lock);
@@ -181,7 +195,8 @@ ThreadSignalEvent::Fire()
 	if (error != B_OK || !fSignal->IsPending())
 		fSignal->SetUnused();
 
-	return error;
+	// We're no longer queued in the DPC queue, so we can be reused.
+	atomic_set(&fPendingDPC, 0);
 }