kernel: Fix deadlock with thread sending signal to itself

UserEvent can be fired from scheduler_reschedule() i.e. while holding current
thread scheduler_lock. If the current thread goes sleep and during reschedule
one of its timers sends a signel to it, then scheduler_enqueue_in_run_queue()
attempts to acquire again its scheduler_lock resulting in a deadlock.

There was also a minor issue with both scheduler_reschedule() and
scheduler_enqueue_in_run_queue() acquiring current CPU scheduler mode lock.

headers/private/kernel/UserEvent.h

@@ -28,11 +28,13 @@ struct UserEvent {
 };
 
 
-struct SignalEvent : UserEvent {
+struct SignalEvent : UserEvent, private DPCCallback {
 	virtual						~SignalEvent();
 
 			void				SetUserValue(union sigval userValue);
 
+	virtual	status_t			Fire();
+
 protected:
 			struct EventSignal;
 
@@ -41,6 +43,7 @@ protected:
 
 protected:
 			EventSignal*		fSignal;
+			int32				fPendingDPC;
 };
 
 
@@ -48,7 +51,8 @@ struct TeamSignalEvent : SignalEvent {
 	static	TeamSignalEvent*	Create(Team* team, uint32 signalNumber,
 									int32 signalCode, int32 errorCode);
 
-	virtual	status_t			Fire();
+protected:
+	virtual	void				DoDPC(DPCQueue* queue);
 
 private:
 								TeamSignalEvent(Team* team,
@@ -64,7 +68,8 @@ struct ThreadSignalEvent : SignalEvent {
 									int32 signalCode, int32 errorCode,
 									pid_t sendingTeam);
 
-	virtual	status_t			Fire();
+protected:
+	virtual	void				DoDPC(DPCQueue* queue);
 
 private:
 								ThreadSignalEvent(Thread* thread,

src/system/kernel/UserEvent.cpp

@@ -56,7 +56,8 @@ private:
 
 SignalEvent::SignalEvent(EventSignal* signal)
 	:
-	fSignal(signal)
+	fSignal(signal),
+	fPendingDPC(0)
 {
 }
 
@@ -74,6 +75,24 @@ SignalEvent::SetUserValue(union sigval userValue)
 }
 
 
+status_t
+SignalEvent::Fire()
+{
+	bool wasPending = atomic_get_and_set(&fPendingDPC, 1) != 0;
+	if (wasPending)
+		return B_BUSY;
+
+	if (fSignal->MarkUsed()) {
+		atomic_set(&fPendingDPC, 0);
+		return B_BUSY;
+	}
+
+	DPCQueue::DefaultQueue(B_NORMAL_PRIORITY)->Add(this);
+
+	return B_OK;
+}
+
+
 // #pragma mark - TeamSignalEvent
 
 
@@ -106,12 +125,9 @@ TeamSignalEvent::Create(Team* team, uint32 signalNumber, int32 signalCode,
 }
 
 
-status_t
-TeamSignalEvent::Fire()
+void
+TeamSignalEvent::DoDPC(DPCQueue* queue)
 {
-	if (fSignal->MarkUsed())
-		return B_BUSY;
-
 	fSignal->AcquireReference();
 		// one reference is transferred to send_signal_to_team_locked
 
@@ -125,7 +141,8 @@ TeamSignalEvent::Fire()
 	if (error != B_OK || !fSignal->IsPending())
 		fSignal->SetUnused();
 
-	return error;
+	// We're no longer queued in the DPC queue, so we can be reused.
+	atomic_set(&fPendingDPC, 0);
 }
 
 
@@ -161,12 +178,9 @@ ThreadSignalEvent::Create(Thread* thread, uint32 signalNumber, int32 signalCode,
 }
 
 
-status_t
-ThreadSignalEvent::Fire()
+void
+ThreadSignalEvent::DoDPC(DPCQueue* queue)
 {
-	if (fSignal->MarkUsed())
-		return B_BUSY;
-
 	fSignal->AcquireReference();
 		// one reference is transferred to send_signal_to_team_locked
 	InterruptsReadSpinLocker teamLocker(fThread->team_lock);
@@ -181,7 +195,8 @@ ThreadSignalEvent::Fire()
 	if (error != B_OK || !fSignal->IsPending())
 		fSignal->SetUnused();
 
-	return error;
+	// We're no longer queued in the DPC queue, so we can be reused.
+	atomic_set(&fPendingDPC, 0);
 }